diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 064e44f94..d5b2d2f47 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -72,6 +72,8 @@ add_library(core STATIC file_sys/delay_generator.h file_sys/ivfc_archive.cpp file_sys/ivfc_archive.h + file_sys/layered_fs.cpp + file_sys/layered_fs.h file_sys/ncch_container.cpp file_sys/ncch_container.h file_sys/patch.cpp diff --git a/src/core/file_sys/layered_fs.cpp b/src/core/file_sys/layered_fs.cpp new file mode 100644 index 000000000..9a194569a --- /dev/null +++ b/src/core/file_sys/layered_fs.cpp @@ -0,0 +1,551 @@ +// Copyright 2020 Citra Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#include +#include +#include "common/alignment.h" +#include "common/assert.h" +#include "common/common_paths.h" +#include "common/file_util.h" +#include "common/string_util.h" +#include "common/swap.h" +#include "core/file_sys/layered_fs.h" +#include "core/file_sys/patch.h" + +namespace FileSys { + +struct FileRelocationInfo { + int type; // 0 - none, 1 - replaced / created, 2 - patched, 3 - removed + u64 original_offset; // Type 0. Offset is absolute + FileUtil::IOFile replace_file; // Type 1 + std::vector patched_file; // Type 2 + u64 size; // Relocated file size +}; +struct LayeredFS::File { + std::string name; + std::string path; + FileRelocationInfo relocation{}; + Directory* parent; +}; + +struct DirectoryMetadata { + u32_le parent_directory_offset; + u32_le next_sibling_offset; + u32_le first_child_directory_offset; + u32_le first_file_offset; + u32_le hash_bucket_next; + u32_le name_length; + // Followed by a name of name length (aligned up to 4) +}; +static_assert(sizeof(DirectoryMetadata) == 0x18, "Size of DirectoryMetadata is not correct"); + +struct FileMetadata { + u32_le parent_directory_offset; + u32_le next_sibling_offset; + u64_le file_data_offset; + u64_le file_data_length; + u32_le hash_bucket_next; + u32_le name_length; + // Followed by a name of name length (aligned up to 4) +}; +static_assert(sizeof(FileMetadata) == 0x20, "Size of FileMetadata is not correct"); + +LayeredFS::LayeredFS(std::shared_ptr romfs_, std::string patch_path_, + std::string patch_ext_path_) + : romfs(std::move(romfs_)), patch_path(std::move(patch_path_)), + patch_ext_path(std::move(patch_ext_path_)) { + + romfs->ReadFile(0, sizeof(header), reinterpret_cast(&header)); + + ASSERT_MSG(header.header_length == sizeof(header), "Header size is incorrect"); + + // TODO: is root always the first directory in table? + root.parent = &root; + LoadDirectory(root, 0); + + LoadRelocations(); + LoadExtRelocations(); + + RebuildMetadata(); +} + +LayeredFS::~LayeredFS() = default; + +void LayeredFS::LoadDirectory(Directory& current, u32 offset) { + DirectoryMetadata metadata; + romfs->ReadFile(header.directory_metadata_table.offset + offset, sizeof(metadata), + reinterpret_cast(&metadata)); + + current.name = ReadName(header.directory_metadata_table.offset + offset + sizeof(metadata), + metadata.name_length); + current.path = current.parent->path + current.name + DIR_SEP; + directory_path_map.emplace(current.path, ¤t); + + if (metadata.first_file_offset != 0xFFFFFFFF) { + LoadFile(current, metadata.first_file_offset); + } + + if (metadata.first_child_directory_offset != 0xFFFFFFFF) { + auto child = std::make_unique(); + auto& directory = *child; + directory.parent = ¤t; + current.directories.emplace_back(std::move(child)); + LoadDirectory(directory, metadata.first_child_directory_offset); + } + + if (metadata.next_sibling_offset != 0xFFFFFFFF) { + auto sibling = std::make_unique(); + auto& directory = *sibling; + directory.parent = current.parent; + current.parent->directories.emplace_back(std::move(sibling)); + LoadDirectory(directory, metadata.next_sibling_offset); + } +} + +void LayeredFS::LoadFile(Directory& parent, u32 offset) { + FileMetadata metadata; + romfs->ReadFile(header.file_metadata_table.offset + offset, sizeof(metadata), + reinterpret_cast(&metadata)); + + auto file = std::make_unique(); + file->name = ReadName(header.file_metadata_table.offset + offset + sizeof(metadata), + metadata.name_length); + file->path = parent.path + file->name; + file->relocation.original_offset = header.file_data_offset + metadata.file_data_offset; + file->relocation.size = metadata.file_data_length; + file->parent = &parent; + + file_path_map.emplace(file->path, file.get()); + parent.files.emplace_back(std::move(file)); + + if (metadata.next_sibling_offset != 0xFFFFFFFF) { + LoadFile(parent, metadata.next_sibling_offset); + } +} + +std::string LayeredFS::ReadName(u32 offset, u32 name_length) { + std::vector buffer(name_length / sizeof(u16_le)); + romfs->ReadFile(offset, name_length, reinterpret_cast(buffer.data())); + + std::u16string name(buffer.size(), 0); + std::transform(buffer.begin(), buffer.end(), name.begin(), [](u16_le character) { + return static_cast(static_cast(character)); + }); + return Common::UTF16ToUTF8(name); +} + +void LayeredFS::LoadRelocations() { + if (!FileUtil::Exists(patch_path)) { + return; + } + + const FileUtil::DirectoryEntryCallable callback = [this, + &callback](u64* /*num_entries_out*/, + const std::string& directory, + const std::string& virtual_name) { + auto* parent = directory_path_map.at(directory.substr(patch_path.size() - 1)); + + if (FileUtil::IsDirectory(directory + virtual_name + DIR_SEP)) { + const auto path = (directory + virtual_name + DIR_SEP).substr(patch_path.size() - 1); + if (!directory_path_map.count(path)) { // Add this directory + auto directory = std::make_unique(); + directory->name = virtual_name; + directory->path = path; + directory->parent = parent; + directory_path_map.emplace(path, directory.get()); + parent->directories.emplace_back(std::move(directory)); + LOG_INFO(Service_FS, "LayeredFS created directory {}", path); + } + return FileUtil::ForeachDirectoryEntry(nullptr, directory + virtual_name + DIR_SEP, + callback); + } + + const auto path = (directory + virtual_name).substr(patch_path.size() - 1); + if (!file_path_map.count(path)) { // Newly created file + auto file = std::make_unique(); + file->name = virtual_name; + file->path = path; + file->parent = parent; + file_path_map.emplace(path, file.get()); + parent->files.emplace_back(std::move(file)); + LOG_INFO(Service_FS, "LayeredFS created file {}", path); + } + + auto* file = file_path_map.at(path); + file->relocation.replace_file = FileUtil::IOFile(directory + virtual_name, "rb"); + if (file->relocation.replace_file) { + file->relocation.type = 1; + file->relocation.size = file->relocation.replace_file.GetSize(); + LOG_INFO(Service_FS, "LayeredFS replacement file in use for {}", path); + } else { + LOG_ERROR(Service_FS, "Could not open replacement file for {}", path); + } + return true; + }; + + FileUtil::ForeachDirectoryEntry(nullptr, patch_path, callback); +} + +void LayeredFS::LoadExtRelocations() { + if (!FileUtil::Exists(patch_ext_path)) { + return; + } + + if (patch_ext_path.back() == '/' || patch_ext_path.back() == '\\') { + // ScanDirectoryTree expects a path without trailing '/' + patch_ext_path.erase(patch_ext_path.size() - 1, 1); + } + + FileUtil::FSTEntry result; + FileUtil::ScanDirectoryTree(patch_ext_path, result, 256); + + for (const auto& entry : result.children) { + if (FileUtil::IsDirectory(entry.physicalName)) { + continue; + } + + const auto path = entry.physicalName.substr(patch_ext_path.size()); + if (path.size() >= 5 && path.substr(path.size() - 5) == ".stub") { + // Remove the corresponding file if exists + const auto file_path = path.substr(0, path.size() - 5); + if (file_path_map.count(file_path)) { + auto& file = *file_path_map[file_path]; + file.relocation.type = 3; + file.relocation.size = 0; + file_path_map.erase(file_path); + LOG_INFO(Service_FS, "LayeredFS removed file {}", file_path); + } else { + LOG_WARNING(Service_FS, "LayeredFS file for stub {} not found", path); + } + } else if (path.size() >= 4) { + const auto extension = path.substr(path.size() - 4); + if (extension != ".ips" && extension != ".bps") { + LOG_WARNING(Service_FS, "LayeredFS unknown ext file {}", path); + } + + const auto file_path = path.substr(0, path.size() - 4); + if (!file_path_map.count(file_path)) { + LOG_WARNING(Service_FS, "LayeredFS original file for patch {} not found", path); + continue; + } + + FileUtil::IOFile patch_file(entry.physicalName, "rb"); + if (!patch_file) { + LOG_ERROR(Service_FS, "LayeredFS Could not open file {}", entry.physicalName); + continue; + } + + const auto size = patch_file.GetSize(); + std::vector patch(size); + if (patch_file.ReadBytes(patch.data(), size) != size) { + LOG_ERROR(Service_FS, "LayeredFS Could not read file {}", entry.physicalName); + continue; + } + + auto& file = *file_path_map[file_path]; + std::vector buffer(file.relocation.size); // Original size + romfs->ReadFile(file.relocation.original_offset, buffer.size(), buffer.data()); + + bool ret = false; + if (extension == ".ips") { + ret = Patch::ApplyIpsPatch(patch, buffer); + } else { + ret = Patch::ApplyBpsPatch(patch, buffer); + } + + if (ret) { + LOG_INFO(Service_FS, "LayeredFS patched file {}", file_path); + + file.relocation.type = 2; + file.relocation.size = buffer.size(); + file.relocation.patched_file = std::move(buffer); + } else { + LOG_ERROR(Service_FS, "LayeredFS failed to patch file {}", file_path); + } + } else { + LOG_WARNING(Service_FS, "LayeredFS unknown ext file {}", path); + } + } +} + +std::size_t GetNameSize(const std::string& name) { + std::u16string u16name = Common::UTF8ToUTF16(name); + return Common::AlignUp(u16name.size() * 2, 4); +} + +void LayeredFS::PrepareBuildDirectory(Directory& current) { + directory_metadata_offset_map.emplace(¤t, current_directory_offset); + directory_list.emplace_back(¤t); + current_directory_offset += sizeof(DirectoryMetadata) + GetNameSize(current.name); +} + +void LayeredFS::PrepareBuildFile(File& current) { + if (current.relocation.type == 3) { // Deleted files are not counted + return; + } + file_metadata_offset_map.emplace(¤t, current_file_offset); + file_list.emplace_back(¤t); + current_file_offset += sizeof(FileMetadata) + GetNameSize(current.name); +} + +void LayeredFS::PrepareBuild(Directory& current) { + for (const auto& child : current.files) { + PrepareBuildFile(*child); + } + + for (const auto& child : current.directories) { + PrepareBuildDirectory(*child); + } + + for (const auto& child : current.directories) { + PrepareBuild(*child); + } +} + +// Implementation from 3dbrew +u32 CalcHash(const std::string& name, u32 parent_offset) { + u32 hash = parent_offset ^ 123456789; + + std::u16string u16name = Common::UTF8ToUTF16(name); + std::vector tmp_buffer(u16name.size()); + std::transform(u16name.begin(), u16name.end(), tmp_buffer.begin(), [](char16_t character) { + return static_cast(static_cast(character)); + }); + + std::vector buffer(tmp_buffer.size() * 2); + std::memcpy(buffer.data(), tmp_buffer.data(), buffer.size()); + for (std::size_t i = 0; i < buffer.size(); i += 2) { + hash = (hash >> 5) | (hash << 27); + hash ^= static_cast((buffer[i]) | (buffer[i + 1] << 8)); + } + return hash; +} + +std::size_t WriteName(u8* dest, std::u16string name) { + const auto buffer_size = Common::AlignUp(name.size() * 2, 4); + std::vector buffer(buffer_size / 2); + std::transform(name.begin(), name.end(), buffer.begin(), [](char16_t character) { + return static_cast(static_cast(character)); + }); + std::memcpy(dest, buffer.data(), buffer_size); + + return buffer_size; +} + +void LayeredFS::BuildDirectories() { + directory_metadata_table.resize(current_directory_offset, 0xFF); + + std::size_t written = 0; + for (const auto& directory : directory_list) { + DirectoryMetadata metadata; + std::memset(&metadata, 0xFF, sizeof(metadata)); + metadata.parent_directory_offset = directory_metadata_offset_map.at(directory->parent); + + if (directory->parent != directory) { + bool flag = false; + for (const auto& sibling : directory->parent->directories) { + if (flag) { + metadata.next_sibling_offset = directory_metadata_offset_map.at(sibling.get()); + break; + } else if (sibling.get() == directory) { + flag = true; + } + } + } + + if (!directory->directories.empty()) { + metadata.first_child_directory_offset = + directory_metadata_offset_map.at(directory->directories.front().get()); + } + + if (!directory->files.empty()) { + metadata.first_file_offset = + file_metadata_offset_map.at(directory->files.front().get()); + } + + const auto bucket = CalcHash(directory->name, metadata.parent_directory_offset) % + directory_hash_table.size(); + metadata.hash_bucket_next = directory_hash_table[bucket]; + directory_hash_table[bucket] = directory_metadata_offset_map.at(directory); + + // Write metadata and name + std::u16string u16name = Common::UTF8ToUTF16(directory->name); + metadata.name_length = u16name.size() * 2; + + std::memcpy(directory_metadata_table.data() + written, &metadata, sizeof(metadata)); + written += sizeof(metadata); + + written += WriteName(directory_metadata_table.data() + written, u16name); + } + + ASSERT_MSG(written == directory_metadata_table.size(), + "Calculated size for directory metadata table is wrong"); +} + +void LayeredFS::BuildFiles() { + file_metadata_table.resize(current_file_offset, 0xFF); + + std::size_t written = 0; + for (const auto& file : file_list) { + FileMetadata metadata; + std::memset(&metadata, 0xFF, sizeof(metadata)); + + metadata.parent_directory_offset = directory_metadata_offset_map.at(file->parent); + + bool flag = false; + for (const auto& sibling : file->parent->files) { + if (sibling->relocation.type == 3) { // removed file + continue; + } + if (flag) { + metadata.next_sibling_offset = file_metadata_offset_map.at(sibling.get()); + break; + } else if (sibling.get() == file) { + flag = true; + } + } + + metadata.file_data_offset = current_data_offset; + metadata.file_data_length = file->relocation.size; + current_data_offset += Common::AlignUp(metadata.file_data_length, 16); + if (metadata.file_data_length != 0) { + data_offset_map.emplace(metadata.file_data_offset, file); + } + + const auto bucket = + CalcHash(file->name, metadata.parent_directory_offset) % file_hash_table.size(); + metadata.hash_bucket_next = file_hash_table[bucket]; + file_hash_table[bucket] = file_metadata_offset_map.at(file); + + // Write metadata and name + std::u16string u16name = Common::UTF8ToUTF16(file->name); + metadata.name_length = u16name.size() * 2; + + std::memcpy(file_metadata_table.data() + written, &metadata, sizeof(metadata)); + written += sizeof(metadata); + + written += WriteName(file_metadata_table.data() + written, u16name); + } + + ASSERT_MSG(written == file_metadata_table.size(), + "Calculated size for file metadata table is wrong"); +} + +// Implementation from 3dbrew +std::size_t GetHashTableSize(std::size_t entry_count) { + if (entry_count < 3) { + return 3; + } else if (entry_count < 19) { + return entry_count | 1; + } else { + std::size_t count = entry_count; + while (count % 2 == 0 || count % 3 == 0 || count % 5 == 0 || count % 7 == 0 || + count % 11 == 0 || count % 13 == 0 || count % 17 == 0) { + count++; + } + return count; + } +} + +void LayeredFS::RebuildMetadata() { + PrepareBuildDirectory(root); + PrepareBuild(root); + + directory_hash_table.resize(GetHashTableSize(directory_list.size()), 0xFFFFFFFF); + file_hash_table.resize(GetHashTableSize(file_list.size()), 0xFFFFFFFF); + + BuildDirectories(); + BuildFiles(); + + // Create header + RomFSHeader header; + header.header_length = sizeof(header); + header.directory_hash_table = { + /*offset*/ sizeof(header), + /*length*/ static_cast(directory_hash_table.size() * sizeof(u32_le))}; + header.directory_metadata_table = { + /*offset*/ + header.directory_hash_table.offset + header.directory_hash_table.length, + /*length*/ static_cast(directory_metadata_table.size())}; + header.file_hash_table = { + /*offset*/ + header.directory_metadata_table.offset + header.directory_metadata_table.length, + /*length*/ static_cast(file_hash_table.size() * sizeof(u32_le))}; + header.file_metadata_table = {/*offset*/ header.file_hash_table.offset + + header.file_hash_table.length, + /*length*/ static_cast(file_metadata_table.size())}; + header.file_data_offset = + Common::AlignUp(header.file_metadata_table.offset + header.file_metadata_table.length, 16); + + // Write hash table and metadata table + metadata.resize(header.file_data_offset); + std::memcpy(metadata.data(), &header, header.header_length); + std::memcpy(metadata.data() + header.directory_hash_table.offset, directory_hash_table.data(), + header.directory_hash_table.length); + std::memcpy(metadata.data() + header.directory_metadata_table.offset, + directory_metadata_table.data(), header.directory_metadata_table.length); + std::memcpy(metadata.data() + header.file_hash_table.offset, file_hash_table.data(), + header.file_hash_table.length); + std::memcpy(metadata.data() + header.file_metadata_table.offset, file_metadata_table.data(), + header.file_metadata_table.length); +} + +std::size_t LayeredFS::GetSize() const { + return metadata.size() + current_data_offset; +} + +std::size_t LayeredFS::ReadFile(std::size_t offset, std::size_t length, u8* buffer) { + ASSERT_MSG(offset + length <= GetSize(), "Out of bound"); + + std::size_t read_size = 0; + if (offset < metadata.size()) { + // First read the metadata + const auto to_read = std::min(metadata.size() - offset, length); + std::memcpy(buffer, metadata.data() + offset, to_read); + read_size += to_read; + offset = 0; + } else { + offset -= metadata.size(); + } + + // Read files + auto current = (--data_offset_map.upper_bound(offset)); + while (read_size < length) { + const auto relative_offset = offset - current->first; + std::size_t to_read{}; + if (current->second->relocation.size > relative_offset) { + to_read = + std::min(current->second->relocation.size - relative_offset, length - read_size); + } + const auto alignment = + std::min(Common::AlignUp(current->second->relocation.size, 16) - relative_offset, + length - read_size) - + to_read; + + // Read the file in different ways depending on relocation type + auto& relocation = current->second->relocation; + if (relocation.type == 0) { // none + romfs->ReadFile(relocation.original_offset + relative_offset, to_read, + buffer + read_size); + } else if (relocation.type == 1) { // replace + relocation.replace_file.Seek(relative_offset, SEEK_SET); + relocation.replace_file.ReadBytes(buffer + read_size, to_read); + } else if (relocation.type == 2) { // patch + std::memcpy(buffer + read_size, relocation.patched_file.data() + relative_offset, + to_read); + } else { + UNREACHABLE(); + } + + std::memset(buffer + read_size + to_read, 0, alignment); + + read_size += to_read + alignment; + offset += to_read + alignment; + current++; + } + + return read_size; +} + +} // namespace FileSys diff --git a/src/core/file_sys/layered_fs.h b/src/core/file_sys/layered_fs.h new file mode 100644 index 000000000..b9dcb831f --- /dev/null +++ b/src/core/file_sys/layered_fs.h @@ -0,0 +1,117 @@ +// Copyright 2020 Citra Emulator Project +// Licensed under GPLv2 or any later version +// Refer to the license.txt file included. + +#pragma once + +#include +#include +#include +#include +#include +#include "common/common_types.h" +#include "common/swap.h" +#include "core/file_sys/romfs_reader.h" + +namespace FileSys { + +struct RomFSHeader { + struct Descriptor { + u32_le offset; + u32_le length; + }; + u32_le header_length; + Descriptor directory_hash_table; + Descriptor directory_metadata_table; + Descriptor file_hash_table; + Descriptor file_metadata_table; + u32_le file_data_offset; +}; +static_assert(sizeof(RomFSHeader) == 0x28, "Size of RomFSHeader is not correct"); + +/** + * LayeredFS implementation. This basically adds a layer to another RomFSReader. + * + * patch_path: Path for RomFS replacements. Files present in this path replace or create + * corresponding files in RomFS. + * patch_ext_path: Path for RomFS extensions. Files present in this path: + * - When with an extension of ".stub", remove the corresponding file in the RomFS. + * - When with an extension of ".ips" or ".bps", patch the file in the RomFS. + */ +class LayeredFS : public RomFSReader { +public: + explicit LayeredFS(std::shared_ptr romfs, std::string patch_path, + std::string patch_ext_path); + ~LayeredFS(); + + std::size_t GetSize() const override; + std::size_t ReadFile(std::size_t offset, std::size_t length, u8* buffer) override; + +private: + struct File; + struct Directory { + std::string name; + std::string path; // with trailing '/' + std::vector> files; + std::vector> directories; + Directory* parent; + }; + + std::string ReadName(u32 offset, u32 name_length); + + // Loads the current directory, then its siblings, and then its children. + void LoadDirectory(Directory& current, u32 offset); + + // Load the file at offset, and then its siblings. + void LoadFile(Directory& parent, u32 offset); + + // Load replace/create relocations + void LoadRelocations(); + + // Load patch/remove relocations + void LoadExtRelocations(); + + // Calculate the offset of a single directory add it to the map and list of directories + void PrepareBuildDirectory(Directory& current); + + // Calculate the offset of a single file add it to the map and list of files + void PrepareBuildFile(File& current); + + // Recursively generate a sequence of files and directories and their offsets for all + // children of current. (The current directory itself is not handled.) + void PrepareBuild(Directory& current); + + void BuildDirectories(); + void BuildFiles(); + + void RebuildMetadata(); + + std::shared_ptr romfs; + std::string patch_path; + std::string patch_ext_path; + + RomFSHeader header; + Directory root; + std::unordered_map file_path_map; + std::unordered_map directory_path_map; + std::map data_offset_map; // assigned data offset -> file + std::vector metadata; // Includes header, hash table and metadata + + // Used for rebuilding header + std::vector directory_hash_table; + std::vector file_hash_table; + + std::unordered_map + directory_metadata_offset_map; // directory -> metadata offset + std::vector directory_list; // sequence of directories to be written to metadata + u64 current_directory_offset{}; // current directory metadata offset + std::vector directory_metadata_table; // rebuilt directory metadata table + + std::unordered_map file_metadata_offset_map; // file -> metadata offset + std::vector file_list; // sequence of files to be written to metadata + u64 current_file_offset{}; // current file metadata offset + std::vector file_metadata_table; // rebuilt file metadata table + u64 current_data_offset{}; // current assigned data offset +}; + +} // namespace FileSys