From 3b7ca4ac354b0d3aad363515defec4b8cb90a092 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 4 Jun 2017 11:22:56 +0200 Subject: [PATCH 1/4] find: Improve debug log --- src/cmds/restic/cmd_find.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cmds/restic/cmd_find.go b/src/cmds/restic/cmd_find.go index e78b14e2d..853fc8815 100644 --- a/src/cmds/restic/cmd_find.go +++ b/src/cmds/restic/cmd_find.go @@ -173,7 +173,7 @@ func (s *statefulOutput) Finish() { } func findInTree(repo *repository.Repository, pat *findPattern, id restic.ID, prefix string, state *statefulOutput) error { - debug.Log("checking tree %v\n", id) + debug.Log("%v checking tree %v\n", prefix, id) tree, err := repo.LoadTree(id) if err != nil { From 7b5efaf7b03ed7830acff7560b25e80ce62c8788 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 4 Jun 2017 11:38:46 +0200 Subject: [PATCH 2/4] find: Move functions to struct --- src/cmds/restic/cmd_find.go | 53 ++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/src/cmds/restic/cmd_find.go b/src/cmds/restic/cmd_find.go index 853fc8815..102ebecfc 100644 --- a/src/cmds/restic/cmd_find.go +++ b/src/cmds/restic/cmd_find.go @@ -12,7 +12,6 @@ import ( "restic" "restic/debug" "restic/errors" - "restic/repository" ) var cmdFind = &cobra.Command{ @@ -172,10 +171,17 @@ func (s *statefulOutput) Finish() { } } -func findInTree(repo *repository.Repository, pat *findPattern, id restic.ID, prefix string, state *statefulOutput) error { - debug.Log("%v checking tree %v\n", prefix, id) +// Finder bundles information needed to find a file or directory. +type Finder struct { + repo restic.Repository + pat findPattern + out statefulOutput +} - tree, err := repo.LoadTree(id) +func (f *Finder) findInTree(treeID restic.ID, prefix string) error { + debug.Log("%v checking tree %v\n", prefix, treeID.Str()) + + tree, err := f.repo.LoadTree(treeID) if err != nil { return err } @@ -184,34 +190,32 @@ func findInTree(repo *repository.Repository, pat *findPattern, id restic.ID, pre debug.Log(" testing entry %q\n", node.Name) name := node.Name - if pat.ignoreCase { + if f.pat.ignoreCase { name = strings.ToLower(name) } - m, err := filepath.Match(pat.pattern, name) + m, err := filepath.Match(f.pat.pattern, name) if err != nil { return err } if m { - debug.Log(" pattern matches\n") - if !pat.oldest.IsZero() && node.ModTime.Before(pat.oldest) { - debug.Log(" ModTime is older than %s\n", pat.oldest) + if !f.pat.oldest.IsZero() && node.ModTime.Before(f.pat.oldest) { + debug.Log(" ModTime is older than %s\n", f.pat.oldest) continue } - if !pat.newest.IsZero() && node.ModTime.After(pat.newest) { - debug.Log(" ModTime is newer than %s\n", pat.newest) + if !f.pat.newest.IsZero() && node.ModTime.After(f.pat.newest) { + debug.Log(" ModTime is newer than %s\n", f.pat.newest) continue } - state.Print(prefix, node) - } else { - debug.Log(" pattern does not match\n") + debug.Log(" found match\n") + f.out.Print(prefix, node) } if node.Type == "dir" { - if err := findInTree(repo, pat, *node.Subtree, filepath.Join(prefix, node.Name), state); err != nil { + if err := f.findInTree(*node.Subtree, filepath.Join(prefix, node.Name)); err != nil { return err } } @@ -220,11 +224,11 @@ func findInTree(repo *repository.Repository, pat *findPattern, id restic.ID, pre return nil } -func findInSnapshot(repo *repository.Repository, sn *restic.Snapshot, pat findPattern, state *statefulOutput) error { - debug.Log("searching in snapshot %s\n for entries within [%s %s]", sn.ID(), pat.oldest, pat.newest) +func (f *Finder) findInSnapshot(sn *restic.Snapshot) error { + debug.Log("searching in snapshot %s\n for entries within [%s %s]", sn.ID(), f.pat.oldest, f.pat.newest) - state.newsn = sn - if err := findInTree(repo, &pat, *sn.Tree, string(filepath.Separator), state); err != nil { + f.out.newsn = sn + if err := f.findInTree(*sn.Tree, string(filepath.Separator)); err != nil { return err } return nil @@ -273,13 +277,18 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error { ctx, cancel := context.WithCancel(gopts.ctx) defer cancel() - state := statefulOutput{ListLong: opts.ListLong, JSON: globalOptions.JSON} + + f := &Finder{ + repo: repo, + pat: pat, + out: statefulOutput{ListLong: opts.ListLong, JSON: globalOptions.JSON}, + } for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, opts.Snapshots) { - if err = findInSnapshot(repo, sn, pat, &state); err != nil { + if err = f.findInSnapshot(sn); err != nil { return err } } - state.Finish() + f.out.Finish() return nil } From a90e0c65957195ceb50dddb053e7242a6b56e965 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 4 Jun 2017 11:42:40 +0200 Subject: [PATCH 3/4] find: Check trees only once --- src/cmds/restic/cmd_find.go | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/cmds/restic/cmd_find.go b/src/cmds/restic/cmd_find.go index 102ebecfc..70d1cc127 100644 --- a/src/cmds/restic/cmd_find.go +++ b/src/cmds/restic/cmd_find.go @@ -173,12 +173,18 @@ func (s *statefulOutput) Finish() { // Finder bundles information needed to find a file or directory. type Finder struct { - repo restic.Repository - pat findPattern - out statefulOutput + repo restic.Repository + pat findPattern + out statefulOutput + notfound restic.IDSet } func (f *Finder) findInTree(treeID restic.ID, prefix string) error { + if f.notfound.Has(treeID) { + debug.Log("%v skipping tree %v, has already been checked", prefix, treeID.Str()) + return nil + } + debug.Log("%v checking tree %v\n", prefix, treeID.Str()) tree, err := f.repo.LoadTree(treeID) @@ -186,6 +192,7 @@ func (f *Finder) findInTree(treeID restic.ID, prefix string) error { return err } + var found bool for _, node := range tree.Nodes { debug.Log(" testing entry %q\n", node.Name) @@ -211,6 +218,7 @@ func (f *Finder) findInTree(treeID restic.ID, prefix string) error { } debug.Log(" found match\n") + found = true f.out.Print(prefix, node) } @@ -221,6 +229,10 @@ func (f *Finder) findInTree(treeID restic.ID, prefix string) error { } } + if !found { + f.notfound.Insert(treeID) + } + return nil } @@ -279,9 +291,10 @@ func runFind(opts FindOptions, gopts GlobalOptions, args []string) error { defer cancel() f := &Finder{ - repo: repo, - pat: pat, - out: statefulOutput{ListLong: opts.ListLong, JSON: globalOptions.JSON}, + repo: repo, + pat: pat, + out: statefulOutput{ListLong: opts.ListLong, JSON: globalOptions.JSON}, + notfound: restic.NewIDSet(), } for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, opts.Snapshots) { if err = f.findInSnapshot(sn); err != nil { From 9cd664caa3e9d8803a7e6379f0488338b8face91 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 4 Jun 2017 11:50:38 +0200 Subject: [PATCH 4/4] Add entry to CHANGELOG --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8fffc1e18..92274a1be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,12 @@ Important Changes in 0.X.Y https://github.com/restic/restic/issues/512 https://github.com/restic/restic/pull/978 + * Improved performance for the `find` command: Restic recognizes paths it has + already checked for the files in question, so the number of backend requests + is reduced a lot. + https://github.com/restic/restic/issues/989 + https://github.com/restic/restic/pull/993 + Important Changes in 0.6.1 ==========================