diff --git a/src/restic/find.go b/src/restic/find.go index baeb47ca9..7d7a8698a 100644 --- a/src/restic/find.go +++ b/src/restic/find.go @@ -5,8 +5,35 @@ import ( "restic/repository" ) +// FindUsedBlobs traverse the tree ID and adds all seen blobs to blobs. +func findUsedBlobs(repo *repository.Repository, treeID backend.ID, blobs backend.IDSet) error { + blobs.Insert(treeID) + + tree, err := LoadTree(repo, treeID) + if err != nil { + return err + } + + for _, node := range tree.Nodes { + switch node.Type { + case "file": + for _, blob := range node.Content { + blobs.Insert(blob) + } + case "dir": + err := findUsedBlobs(repo, *node.Subtree, blobs) + if err != nil { + return err + } + } + } + + return nil +} + // FindUsedBlobs traverses the tree ID and returns a set of all blobs // encountered. -func FindUsedBlobs(repo *repository.Repository, treeID backend.ID) (backend.IDSet, error) { - return nil, nil +func FindUsedBlobs(repo *repository.Repository, treeID backend.ID) (blobs backend.IDSet, err error) { + blobs = backend.NewIDSet() + return blobs, findUsedBlobs(repo, treeID, blobs) } diff --git a/src/restic/find_test.go b/src/restic/find_test.go index f5973b9ae..759664f29 100644 --- a/src/restic/find_test.go +++ b/src/restic/find_test.go @@ -1,12 +1,69 @@ package restic import ( + "bufio" + "flag" + "fmt" + "os" + "path/filepath" + "sort" "testing" "time" + "restic/backend" "restic/repository" ) +func loadIDSet(t testing.TB, filename string) backend.IDSet { + f, err := os.Open(filename) + if err != nil { + t.Logf("unable to open golden file %v: %v", filename, err) + return backend.IDSet{} + } + + sc := bufio.NewScanner(f) + + ids := backend.NewIDSet() + for sc.Scan() { + id, err := backend.ParseID(sc.Text()) + if err != nil { + t.Errorf("file %v contained invalid id: %v", filename, err) + } + + ids.Insert(id) + } + + if err = f.Close(); err != nil { + t.Errorf("closing file %v failed with error %v", filename, err) + } + + return ids +} + +func saveIDSet(t testing.TB, filename string, s backend.IDSet) { + f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + t.Fatalf("unable to update golden file %v: %v", filename, err) + return + } + + var ids backend.IDs + for id := range s { + ids = append(ids, id) + } + + sort.Sort(ids) + for _, id := range ids { + fmt.Fprintf(f, "%s\n", id) + } + + if err = f.Close(); err != nil { + t.Fatalf("close file %v returned error: %v", filename, err) + } +} + +var updateGoldenFiles = flag.Bool("update", false, "update golden files in testdata/") + const ( testSnapshots = 3 testDepth = 2 @@ -25,7 +82,7 @@ func TestFindUsedBlobs(t *testing.T) { snapshots = append(snapshots, sn) } - for _, sn := range snapshots { + for i, sn := range snapshots { usedBlobs, err := FindUsedBlobs(repo, *sn.Tree) if err != nil { t.Errorf("FindUsedBlobs returned error: %v", err) @@ -37,6 +94,16 @@ func TestFindUsedBlobs(t *testing.T) { continue } - t.Logf("used blobs from snapshot %v (tree %v): %v", sn.ID().Str(), sn.Tree.Str(), usedBlobs) + goldenFilename := filepath.Join("testdata", fmt.Sprintf("used_blobs_snapshot%d", i)) + want := loadIDSet(t, goldenFilename) + + if !want.Equals(usedBlobs) { + t.Errorf("snapshot %d: wrong list of blobs returned:\n missing blobs: %v\n extra blobs: %v", + i, want.Sub(usedBlobs), usedBlobs.Sub(want)) + } + + if *updateGoldenFiles { + saveIDSet(t, goldenFilename, usedBlobs) + } } } diff --git a/src/restic/testdata/used_blobs_snapshot0 b/src/restic/testdata/used_blobs_snapshot0 new file mode 100644 index 000000000..543f534b3 --- /dev/null +++ b/src/restic/testdata/used_blobs_snapshot0 @@ -0,0 +1,37 @@ +087e8d5f45f93a78e52a938ac0b7864f92f8910091c0da69201a156242df3b78 +0bf505951741c44714527d252313b6959ce4f19d2e5512fca1c1b2da14424da3 +0c82d00e6ee78b48559cda2f9cc909beeb8769183b115dfda0a5767832accc8d +2941bfd03b8933bb150b085a2252b69675495af64523bf8d38e67429e7cccb45 +378a9b6862c8fa5c6915f158d16e4416243159bb9da44c564896c065bc6c1cf4 +3ffcf5128fc404c2a363e3e8a8d4c8a7ae8c36fcacba7fdfe71ec9dabcadd567 +40f5ca234e5eed1dc967c83fa99076ef636619148082f300cf877676728ebf14 +42aad1ab6cc964043e53e5da13ed0f2b44a3bf6ae7702f60a805f13028377524 +42bc8f509dbd6b9881cab4c1684d5cf74207046336f654db1b884197f15cae7b +47cf470c1c6de9af00b3b1ee963de8b94f51a2870b3338b3f33cfc565c0f8be4 +587045d0ec69e47a3cc91b13c959aa80add9118ecfac47232ea992650f25f0b9 +615e8851030f318751f3c8baf8fbfa9958e2dd7f25dc1a87dcf6d6f79d1f1a9f +63ec5e835e11203bbeef69095523344dd975f1ab52bdbf4a1db7a53914d967ca +714f9e16404b9ec83de56715e5387b2c4c2ed0af1889166a4e767822f971bf52 +80ba9a145bf46cae605e911c18165c02213e8d11d68dc5b7824f259d17b7b6d0 +86af714d79d18be1c9c0ae23cca9dbd7cef44530e253e80af5bd5c34eab09714 +8a445cf5b6313cbe3b5872a55adde52aa8d1ae188f41d56f176e40a3137ac058 +8e171f7367d1b68012ed1ceec8f54b7b9b8654ebaf63a760017c34d761b17878 +8e98f35e65fb42c85eb4a2ab4793e294148e3f318252cb850a896274d2aa90bc +9d65ba6443863394a8c6582fef4a8aaab2fb46417eef41f1792cdbdb38ee0b4c +9da502ea8e7a768ee0dbafdc613db3df4a7cd9c98af08328265c4d2e953e8efa +9f2899688d2f23391cfd86e7b6d326a54f352bb294160878178639aab4aa378f +a2f3ccf973b3600c06c42dc3b867b263a788c18aa57f4448fea2525b7cbfd784 +b2deaf9174086129ec3b9f79e05401fdb3baf8b75335addffac1950182d779df +b81870ebe27b98f6b8746349e8ea444c96bf2eaac5dbd6236175150ce579f46b +bd4dacd46031b2b837bc9bd06145b0571156fa496408ce728c003ae50b265aaf +c0775cfc822f59524b4ed714d257607fd5f2c9f0dc9f65763a86ffc33aac325b +c3596f717c495d20c33561e991d4295550b6d7544687f2363e999bdc0266224d +c54c4899c4d7dcda8b9e597aebfbaf7d65c9c7a760527d77e7fc9894283d736e +ca51ecf1633896f852929cb2d56ad1b5bed4ab6055bdcf370ced4011bed164aa +ce8b656cead478c34060510962daf97cea52abde68bbef7934dd5c5513cf6f3b +dafbb65569781083b627de833fb931cf98401299a62d747f03d8fc135ab57279 +e193d395410520580e76a5b89b8d23a1d162c0e28c52cb8194d409a74a120f7d +e752efd93f9850ba0cafbbac01bb283c10095ac923cdb8ff027393001123d406 +f728e5576d4ab63248c310396d67d9afa3267dd2dea3cfba690dbd04efe181fb +f75b6460b68d254f2195b08c606672fb55c05fb7bed7e16699b3231104b673ea +fe19f084021bdac5a9a5d270042ff53ef36357dd0743318d0480dee1a43de266 diff --git a/src/restic/testdata/used_blobs_snapshot1 b/src/restic/testdata/used_blobs_snapshot1 new file mode 100644 index 000000000..502e91702 --- /dev/null +++ b/src/restic/testdata/used_blobs_snapshot1 @@ -0,0 +1,34 @@ +011a951a9796979c2b515ef4209662013bd1f16a20a1b35d1d950d7408bdc8b4 +087e8d5f45f93a78e52a938ac0b7864f92f8910091c0da69201a156242df3b78 +0bad18b7f2d82d7c9cf8e405262ad2f3dbe57928aa242c1070b917042a99072d +0bf505951741c44714527d252313b6959ce4f19d2e5512fca1c1b2da14424da3 +0c82d00e6ee78b48559cda2f9cc909beeb8769183b115dfda0a5767832accc8d +2941bfd03b8933bb150b085a2252b69675495af64523bf8d38e67429e7cccb45 +3ffcf5128fc404c2a363e3e8a8d4c8a7ae8c36fcacba7fdfe71ec9dabcadd567 +40f5ca234e5eed1dc967c83fa99076ef636619148082f300cf877676728ebf14 +42bc8f509dbd6b9881cab4c1684d5cf74207046336f654db1b884197f15cae7b +47cf470c1c6de9af00b3b1ee963de8b94f51a2870b3338b3f33cfc565c0f8be4 +4b2e91022c34c756b7bd8ece046a2bab6f0dcad89f46c52d1f84cd48e8da55df +6416bc2321cdeb8758188af2b3925f2c82ffde014bf53b7a69c0f113a5c460fe +714f9e16404b9ec83de56715e5387b2c4c2ed0af1889166a4e767822f971bf52 +80ba9a145bf46cae605e911c18165c02213e8d11d68dc5b7824f259d17b7b6d0 +83bf0196cf45bbca0be7e292688a3622af7888c0e9ec01bb78edaff302cced06 +8a445cf5b6313cbe3b5872a55adde52aa8d1ae188f41d56f176e40a3137ac058 +8e98f35e65fb42c85eb4a2ab4793e294148e3f318252cb850a896274d2aa90bc +907acef01e05c3e0140858423e9284ddd3d64145ba8b0c3293371c5c7ab3d6b7 +9d65ba6443863394a8c6582fef4a8aaab2fb46417eef41f1792cdbdb38ee0b4c +9da502ea8e7a768ee0dbafdc613db3df4a7cd9c98af08328265c4d2e953e8efa +a2f3ccf973b3600c06c42dc3b867b263a788c18aa57f4448fea2525b7cbfd784 +b2deaf9174086129ec3b9f79e05401fdb3baf8b75335addffac1950182d779df +b3915971171e049292e28d7bc61fe362e94f73aa49b578f4ca1322b47d7fc39c +bd4dacd46031b2b837bc9bd06145b0571156fa496408ce728c003ae50b265aaf +c3596f717c495d20c33561e991d4295550b6d7544687f2363e999bdc0266224d +c54c4899c4d7dcda8b9e597aebfbaf7d65c9c7a760527d77e7fc9894283d736e +ca51ecf1633896f852929cb2d56ad1b5bed4ab6055bdcf370ced4011bed164aa +cb8001715217b4f6960aa24c1abb4b60a20c10f23abc1e5f69e0f5436bd788c8 +d39c4c264e01ec47b0386da3775c6b0cc337974627ff55792938cca4895ac6c4 +dafbb65569781083b627de833fb931cf98401299a62d747f03d8fc135ab57279 +e193d395410520580e76a5b89b8d23a1d162c0e28c52cb8194d409a74a120f7d +e791912a7fad8954c764fae41d2958d2feeae2278e403429add9119ab43a36f5 +f728e5576d4ab63248c310396d67d9afa3267dd2dea3cfba690dbd04efe181fb +fe19f084021bdac5a9a5d270042ff53ef36357dd0743318d0480dee1a43de266 diff --git a/src/restic/testdata/used_blobs_snapshot2 b/src/restic/testdata/used_blobs_snapshot2 new file mode 100644 index 000000000..382140b41 --- /dev/null +++ b/src/restic/testdata/used_blobs_snapshot2 @@ -0,0 +1,9 @@ +35e13e123748cd27d1634c4e07e5ff2fc86901b09b215f3125331d1226c782be +378a9b6862c8fa5c6915f158d16e4416243159bb9da44c564896c065bc6c1cf4 +42aad1ab6cc964043e53e5da13ed0f2b44a3bf6ae7702f60a805f13028377524 +47cf470c1c6de9af00b3b1ee963de8b94f51a2870b3338b3f33cfc565c0f8be4 +615e8851030f318751f3c8baf8fbfa9958e2dd7f25dc1a87dcf6d6f79d1f1a9f +83bf0196cf45bbca0be7e292688a3622af7888c0e9ec01bb78edaff302cced06 +9d65ba6443863394a8c6582fef4a8aaab2fb46417eef41f1792cdbdb38ee0b4c +b3915971171e049292e28d7bc61fe362e94f73aa49b578f4ca1322b47d7fc39c +c0775cfc822f59524b4ed714d257607fd5f2c9f0dc9f65763a86ffc33aac325b