diff --git a/restorer.go b/restorer.go
index 8ba401af4..d97bd0a5f 100644
--- a/restorer.go
+++ b/restorer.go
@@ -61,18 +61,12 @@ func (res *Restorer) restoreTo(dst string, dir string, treeID backend.ID) error
 					return err
 				}
 			}
-		}
-	}
 
-	// Restore directory timestamps at the end. If we would do it earlier, restoring files within
-	// those directories would overwrite the timestamp of the directories they are in.
-	for _, node := range tree.Nodes {
-		if node.Type != "dir" {
-			continue
-		}
-
-		if err := node.RestoreTimestamps(filepath.Join(dst, dir, node.Name)); err != nil {
-			return err
+			// Restore directory timestamp at the end. If we would do it earlier, restoring files within
+			// the directory would overwrite the timestamp of the directory they are in.
+			if err := node.RestoreTimestamps(filepath.Join(dst, dir, node.Name)); err != nil {
+				return err
+			}
 		}
 	}