diff --git a/src/video_core/rasterizer_cache.h b/src/video_core/rasterizer_cache.h
index 22987751e9..5236fbb004 100644
--- a/src/video_core/rasterizer_cache.h
+++ b/src/video_core/rasterizer_cache.h
@@ -56,9 +56,27 @@ public:
         last_modified_ticks = cache.GetModifiedTicks();
     }
 
+    void SetMemoryMarked(bool is_memory_marked_) {
+        is_memory_marked = is_memory_marked_;
+    }
+
+    bool IsMemoryMarked() const {
+        return is_memory_marked;
+    }
+
+    void SetSyncPending(bool is_sync_pending_) {
+        is_sync_pending = is_sync_pending_;
+    }
+
+    bool IsSyncPending() const {
+        return is_sync_pending;
+    }
+
 private:
     bool is_registered{};      ///< Whether the object is currently registered with the cache
     bool is_dirty{};           ///< Whether the object is dirty (out of sync with guest memory)
+    bool is_memory_marked{};   ///< Whether it's marking rasterizer memory.
+    bool is_sync_pending{};    ///< Whether it's pending deletion.
     u64 last_modified_ticks{}; ///< When the object was last modified, used for in-order flushing
     VAddr cpu_addr{};          ///< Cpu address memory, unique from emulated virtual address space
 };
@@ -94,6 +112,30 @@ public:
         }
     }
 
+    void OnCPUWrite(VAddr addr, std::size_t size) {
+        std::lock_guard lock{mutex};
+
+        for (const auto& object : GetSortedObjectsFromRegion(addr, size)) {
+            if (object->IsRegistered()) {
+                UnmarkMemory(object);
+                object->SetSyncPending(true);
+                marked_for_unregister.emplace_back(object);
+            }
+        }
+    }
+
+    void SyncGuestHost() {
+        std::lock_guard lock{mutex};
+
+        for (const auto& object : marked_for_unregister) {
+            if (object->IsRegistered()) {
+                object->SetSyncPending(false);
+                Unregister(object);
+            }
+        }
+        marked_for_unregister.clear();
+    }
+
     /// Invalidates everything in the cache
     void InvalidateAll() {
         std::lock_guard lock{mutex};
@@ -120,19 +162,32 @@ protected:
         interval_cache.add({GetInterval(object), ObjectSet{object}});
         map_cache.insert({object->GetCpuAddr(), object});
         rasterizer.UpdatePagesCachedCount(object->GetCpuAddr(), object->GetSizeInBytes(), 1);
+        object->SetMemoryMarked(true);
     }
 
     /// Unregisters an object from the cache
     virtual void Unregister(const T& object) {
         std::lock_guard lock{mutex};
 
+        UnmarkMemory(object);
         object->SetIsRegistered(false);
-        rasterizer.UpdatePagesCachedCount(object->GetCpuAddr(), object->GetSizeInBytes(), -1);
+        if (object->IsSyncPending()) {
+            marked_for_unregister.remove(object);
+            object->SetSyncPending(false);
+        }
         const VAddr addr = object->GetCpuAddr();
         interval_cache.subtract({GetInterval(object), ObjectSet{object}});
         map_cache.erase(addr);
     }
 
+    void UnmarkMemory(const T& object) {
+        if (!object->IsMemoryMarked()) {
+            return;
+        }
+        rasterizer.UpdatePagesCachedCount(object->GetCpuAddr(), object->GetSizeInBytes(), -1);
+        object->SetMemoryMarked(false);
+    }
+
     /// Returns a ticks counter used for tracking when cached objects were last modified
     u64 GetModifiedTicks() {
         std::lock_guard lock{mutex};
@@ -194,4 +249,5 @@ private:
     IntervalCache interval_cache; ///< Cache of objects
     u64 modified_ticks{};         ///< Counter of cache state ticks, used for in-order flushing
     VideoCore::RasterizerInterface& rasterizer;
+    std::list<T> marked_for_unregister;
 };
diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp
index 69dcf952fc..4678914574 100644
--- a/src/video_core/renderer_opengl/gl_rasterizer.cpp
+++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp
@@ -701,15 +701,15 @@ void RasterizerOpenGL::OnCPUWrite(VAddr addr, u64 size) {
         return;
     }
     texture_cache.OnCPUWrite(addr, size);
-    shader_cache.InvalidateRegion(addr, size);
+    shader_cache.OnCPUWrite(addr, size);
     buffer_cache.OnCPUWrite(addr, size);
-    query_cache.InvalidateRegion(addr, size);
 }
 
 void RasterizerOpenGL::SyncGuestHost() {
     MICROPROFILE_SCOPE(OpenGL_CacheManagement);
     texture_cache.SyncGuestHost();
     buffer_cache.SyncGuestHost();
+    shader_cache.SyncGuestHost();
 }
 
 void RasterizerOpenGL::SignalSemaphore(GPUVAddr addr, u32 value) {
diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
index 8b009fc22b..f118e59902 100644
--- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp
+++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp
@@ -532,14 +532,14 @@ void RasterizerVulkan::OnCPUWrite(VAddr addr, u64 size) {
         return;
     }
     texture_cache.OnCPUWrite(addr, size);
-    pipeline_cache.InvalidateRegion(addr, size);
+    pipeline_cache.OnCPUWrite(addr, size);
     buffer_cache.OnCPUWrite(addr, size);
-    query_cache.InvalidateRegion(addr, size);
 }
 
 void RasterizerVulkan::SyncGuestHost() {
     texture_cache.SyncGuestHost();
     buffer_cache.SyncGuestHost();
+    pipeline_cache.SyncGuestHost();
 }
 
 void RasterizerVulkan::SignalSemaphore(GPUVAddr addr, u32 value) {