From 15ad0e5bc72fd034fefad386778c4c2c1851555f Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 19 Aug 2018 12:28:06 +0200 Subject: [PATCH] walk: Pass parent tree ID to WalkFunc --- cmd/restic/cmd_find.go | 6 +-- cmd/restic/cmd_ls.go | 2 +- cmd/restic/cmd_stats.go | 2 +- internal/walker/walker.go | 14 +++---- internal/walker/walker_test.go | 71 ++++++++++++++++++++++++++++++++-- 5 files changed, 80 insertions(+), 15 deletions(-) diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 1f8fc4fb6..5025e9d21 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -258,7 +258,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error } f.out.newsn = sn - return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(nodepath string, node *restic.Node, treeID string, err error) (bool, error) { + return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { if err != nil { return false, err } @@ -338,7 +338,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { } f.out.newsn = sn - return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(nodepath string, node *restic.Node, treeID string, err error) (bool, error) { + return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { if err != nil { return false, err } @@ -379,7 +379,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { f.blobIDs[idStr] = struct{}{} delete(f.blobIDs, idStr[:shortStr]) } - f.out.PrintObject("blob", idStr, nodepath, treeID, sn) + f.out.PrintObject("blob", idStr, nodepath, parentTreeID.String(), sn) break } } diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 7883161ba..64c7ea678 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -203,7 +203,7 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error { for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args[:1]) { printSnapshot(sn) - err := walker.Walk(ctx, repo, *sn.Tree, nil, func(nodepath string, node *restic.Node, treeID string, err error) (bool, error) { + err := walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { if err != nil { return false, err } diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index b22bb5a5e..8718d115d 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -188,7 +188,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest } func statsWalkTree(repo restic.Repository, stats *statsContainer) walker.WalkFunc { - return func(npath string, node *restic.Node, parentTreeID string, nodeErr error) (bool, error) { + return func(_ restic.ID, npath string, node *restic.Node, nodeErr error) (bool, error) { if nodeErr != nil { return true, nodeErr } diff --git a/internal/walker/walker.go b/internal/walker/walker.go index e8c7ae81b..7d6db3abe 100644 --- a/internal/walker/walker.go +++ b/internal/walker/walker.go @@ -33,14 +33,14 @@ var SkipNode = errors.New("skip this node") // tree have ignore set to true, the current tree will not be visited again. // When err is not nil and different from SkipNode, the value returned for // ignore is ignored. -type WalkFunc func(path string, node *restic.Node, parentTreeID string, nodeErr error) (ignore bool, err error) +type WalkFunc func(parentTreeID restic.ID, path string, node *restic.Node, nodeErr error) (ignore bool, err error) // Walk calls walkFn recursively for each node in root. If walkFn returns an // error, it is passed up the call stack. The trees in ignoreTrees are not // walked. If walkFn ignores trees, these are added to the set. func Walk(ctx context.Context, repo TreeLoader, root restic.ID, ignoreTrees restic.IDSet, walkFn WalkFunc) error { tree, err := repo.LoadTree(ctx, root) - _, err = walkFn("/", nil, "", err) + _, err = walkFn(root, "/", nil, err) if err != nil { if err == SkipNode { @@ -53,14 +53,14 @@ func Walk(ctx context.Context, repo TreeLoader, root restic.ID, ignoreTrees rest ignoreTrees = restic.NewIDSet() } - _, err = walk(ctx, repo, "/", root.String(), tree, ignoreTrees, walkFn) + _, err = walk(ctx, repo, "/", root, tree, ignoreTrees, walkFn) return err } // walk recursively traverses the tree, ignoring subtrees when the ID of the // subtree is in ignoreTrees. If err is nil and ignore is true, the subtree ID // will be added to ignoreTrees by walk. -func walk(ctx context.Context, repo TreeLoader, prefix string, treeID string, tree *restic.Tree, ignoreTrees restic.IDSet, walkFn WalkFunc) (ignore bool, err error) { +func walk(ctx context.Context, repo TreeLoader, prefix string, parentTreeID restic.ID, tree *restic.Tree, ignoreTrees restic.IDSet, walkFn WalkFunc) (ignore bool, err error) { var allNodesIgnored = true if len(tree.Nodes) == 0 { @@ -79,7 +79,7 @@ func walk(ctx context.Context, repo TreeLoader, prefix string, treeID string, tr } if node.Type != "dir" { - ignore, err := walkFn(p, node, treeID, nil) + ignore, err := walkFn(parentTreeID, p, node, nil) if err != nil { if err == SkipNode { // skip the remaining entries in this tree @@ -105,7 +105,7 @@ func walk(ctx context.Context, repo TreeLoader, prefix string, treeID string, tr } subtree, err := repo.LoadTree(ctx, *node.Subtree) - ignore, err := walkFn(p, node, treeID, err) + ignore, err := walkFn(parentTreeID, p, node, err) if err != nil { if err == SkipNode { if ignore { @@ -124,7 +124,7 @@ func walk(ctx context.Context, repo TreeLoader, prefix string, treeID string, tr allNodesIgnored = false } - ignore, err = walk(ctx, repo, p, node.Subtree.String(), subtree, ignoreTrees, walkFn) + ignore, err = walk(ctx, repo, p, *node.Subtree, subtree, ignoreTrees, walkFn) if err != nil { return false, err } diff --git a/internal/walker/walker_test.go b/internal/walker/walker_test.go index 67c44b8e2..af5c25f42 100644 --- a/internal/walker/walker_test.go +++ b/internal/walker/walker_test.go @@ -78,7 +78,7 @@ type checkFunc func(t testing.TB) (walker WalkFunc, final func(testing.TB)) func checkItemOrder(want []string) checkFunc { pos := 0 return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { - walker = func(path string, node *restic.Node, treeID string, err error) (bool, error) { + walker = func(treeID restic.ID, path string, node *restic.Node, err error) (bool, error) { if err != nil { t.Errorf("error walking %v: %v", path, err) return false, err @@ -106,13 +106,45 @@ func checkItemOrder(want []string) checkFunc { } } +// checkParentTreeOrder ensures that the order of the 'parentID' arguments is the one passed in as 'want'. +func checkParentTreeOrder(want []string) checkFunc { + pos := 0 + return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { + walker = func(treeID restic.ID, path string, node *restic.Node, err error) (bool, error) { + if err != nil { + t.Errorf("error walking %v: %v", path, err) + return false, err + } + + if pos >= len(want) { + t.Errorf("additional unexpected parent tree ID found: %v", treeID) + return false, nil + } + + if treeID.String() != want[pos] { + t.Errorf("wrong parent tree ID found, want %q, got %q", want[pos], treeID.String()) + } + pos++ + return false, nil + } + + final = func(t testing.TB) { + if pos != len(want) { + t.Errorf("not enough items returned, want %d, got %d", len(want), pos) + } + } + + return walker, final + } +} + // checkSkipFor returns SkipNode if path is in skipFor, it checks that the // paths the walk func is called for are exactly the ones in wantPaths. func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc { var pos int return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { - walker = func(path string, node *restic.Node, treeID string, err error) (bool, error) { + walker = func(treeID restic.ID, path string, node *restic.Node, err error) (bool, error) { if err != nil { t.Errorf("error walking %v: %v", path, err) return false, err @@ -152,7 +184,7 @@ func checkIgnore(skipFor map[string]struct{}, ignoreFor map[string]bool, wantPat var pos int return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { - walker = func(path string, node *restic.Node, treeID string, err error) (bool, error) { + walker = func(treeID restic.ID, path string, node *restic.Node, err error) (bool, error) { if err != nil { t.Errorf("error walking %v: %v", path, err) return false, err @@ -204,6 +236,12 @@ func TestWalker(t *testing.T) { "/subdir", "/subdir/subfile", }), + checkParentTreeOrder([]string{ + "2593e9dba52232c043d68c40d0f9c236b4448e37224941298ea6e223ca1e3a1b", // tree / + "2593e9dba52232c043d68c40d0f9c236b4448e37224941298ea6e223ca1e3a1b", // tree / + "2593e9dba52232c043d68c40d0f9c236b4448e37224941298ea6e223ca1e3a1b", // tree / + "a7f5be55bdd94db9df706a428e0726a4044720c9c94b9ebeb81000debe032087", // tree /subdir + }), checkSkipFor( map[string]struct{}{ "/subdir": struct{}{}, @@ -249,6 +287,16 @@ func TestWalker(t *testing.T) { "/subdir2/subsubdir2", "/subdir2/subsubdir2/subsubfile3", }), + checkParentTreeOrder([]string{ + "31c86f0bc298086b787b5d24e9e33ea566c224be2939ed66a817f7fb6fdba700", // tree / + "31c86f0bc298086b787b5d24e9e33ea566c224be2939ed66a817f7fb6fdba700", // tree / + "31c86f0bc298086b787b5d24e9e33ea566c224be2939ed66a817f7fb6fdba700", // tree / + "af838dc7a83d353f0273c33d93fcdba3220d4517576f09694a971dd23b8e94dc", // tree /subdir1 + "31c86f0bc298086b787b5d24e9e33ea566c224be2939ed66a817f7fb6fdba700", // tree / + "fb749ba6ae01a3814bed9b59d74af8d7593d3074a681d4112c4983d461089e5b", // tree /subdir2 + "fb749ba6ae01a3814bed9b59d74af8d7593d3074a681d4112c4983d461089e5b", // tree /subdir2 + "eb8dd587a9c5e6be87b69d2c5264a19622f75bf6704927aaebaee78d0992531d", // tree /subdir2/subsubdir2 + }), checkSkipFor( map[string]struct{}{ "/subdir1": struct{}{}, @@ -323,6 +371,23 @@ func TestWalker(t *testing.T) { "/subdir3/subfile3", "/zzz other", }), + checkParentTreeOrder([]string{ + "b37368f62fdd6f8f3d19f9ef23c6534988e26db4e5dddc21d206b16b6a17a58f", // tree / + "b37368f62fdd6f8f3d19f9ef23c6534988e26db4e5dddc21d206b16b6a17a58f", // tree / + "b37368f62fdd6f8f3d19f9ef23c6534988e26db4e5dddc21d206b16b6a17a58f", // tree / + "787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir1 + "787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir1 + "787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir1 + "b37368f62fdd6f8f3d19f9ef23c6534988e26db4e5dddc21d206b16b6a17a58f", // tree / + "787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir2 + "787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir2 + "787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir2 + "b37368f62fdd6f8f3d19f9ef23c6534988e26db4e5dddc21d206b16b6a17a58f", // tree / + "787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir3 + "787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir3 + "787b9260d4f0f8298f5cf58945681961982eb6aa1c526845206c5b353aeb4351", // tree /subdir3 + "b37368f62fdd6f8f3d19f9ef23c6534988e26db4e5dddc21d206b16b6a17a58f", // tree / + }), checkIgnore( map[string]struct{}{ "/subdir1": struct{}{},