Add first version of FindUsedBlobs

This commit is contained in:
Alexander Neumann 2016-08-01 18:40:08 +02:00
parent 51b16ad57d
commit ffc3503e6f
5 changed files with 178 additions and 4 deletions

View File

@ -5,8 +5,35 @@ import (
"restic/repository" "restic/repository"
) )
// FindUsedBlobs traverse the tree ID and adds all seen blobs to blobs.
func findUsedBlobs(repo *repository.Repository, treeID backend.ID, blobs backend.IDSet) error {
blobs.Insert(treeID)
tree, err := LoadTree(repo, treeID)
if err != nil {
return err
}
for _, node := range tree.Nodes {
switch node.Type {
case "file":
for _, blob := range node.Content {
blobs.Insert(blob)
}
case "dir":
err := findUsedBlobs(repo, *node.Subtree, blobs)
if err != nil {
return err
}
}
}
return nil
}
// FindUsedBlobs traverses the tree ID and returns a set of all blobs // FindUsedBlobs traverses the tree ID and returns a set of all blobs
// encountered. // encountered.
func FindUsedBlobs(repo *repository.Repository, treeID backend.ID) (backend.IDSet, error) { func FindUsedBlobs(repo *repository.Repository, treeID backend.ID) (blobs backend.IDSet, err error) {
return nil, nil blobs = backend.NewIDSet()
return blobs, findUsedBlobs(repo, treeID, blobs)
} }

View File

@ -1,12 +1,69 @@
package restic package restic
import ( import (
"bufio"
"flag"
"fmt"
"os"
"path/filepath"
"sort"
"testing" "testing"
"time" "time"
"restic/backend"
"restic/repository" "restic/repository"
) )
func loadIDSet(t testing.TB, filename string) backend.IDSet {
f, err := os.Open(filename)
if err != nil {
t.Logf("unable to open golden file %v: %v", filename, err)
return backend.IDSet{}
}
sc := bufio.NewScanner(f)
ids := backend.NewIDSet()
for sc.Scan() {
id, err := backend.ParseID(sc.Text())
if err != nil {
t.Errorf("file %v contained invalid id: %v", filename, err)
}
ids.Insert(id)
}
if err = f.Close(); err != nil {
t.Errorf("closing file %v failed with error %v", filename, err)
}
return ids
}
func saveIDSet(t testing.TB, filename string, s backend.IDSet) {
f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0644)
if err != nil {
t.Fatalf("unable to update golden file %v: %v", filename, err)
return
}
var ids backend.IDs
for id := range s {
ids = append(ids, id)
}
sort.Sort(ids)
for _, id := range ids {
fmt.Fprintf(f, "%s\n", id)
}
if err = f.Close(); err != nil {
t.Fatalf("close file %v returned error: %v", filename, err)
}
}
var updateGoldenFiles = flag.Bool("update", false, "update golden files in testdata/")
const ( const (
testSnapshots = 3 testSnapshots = 3
testDepth = 2 testDepth = 2
@ -25,7 +82,7 @@ func TestFindUsedBlobs(t *testing.T) {
snapshots = append(snapshots, sn) snapshots = append(snapshots, sn)
} }
for _, sn := range snapshots { for i, sn := range snapshots {
usedBlobs, err := FindUsedBlobs(repo, *sn.Tree) usedBlobs, err := FindUsedBlobs(repo, *sn.Tree)
if err != nil { if err != nil {
t.Errorf("FindUsedBlobs returned error: %v", err) t.Errorf("FindUsedBlobs returned error: %v", err)
@ -37,6 +94,16 @@ func TestFindUsedBlobs(t *testing.T) {
continue continue
} }
t.Logf("used blobs from snapshot %v (tree %v): %v", sn.ID().Str(), sn.Tree.Str(), usedBlobs) goldenFilename := filepath.Join("testdata", fmt.Sprintf("used_blobs_snapshot%d", i))
want := loadIDSet(t, goldenFilename)
if !want.Equals(usedBlobs) {
t.Errorf("snapshot %d: wrong list of blobs returned:\n missing blobs: %v\n extra blobs: %v",
i, want.Sub(usedBlobs), usedBlobs.Sub(want))
}
if *updateGoldenFiles {
saveIDSet(t, goldenFilename, usedBlobs)
}
} }
} }

View File

@ -0,0 +1,37 @@
087e8d5f45f93a78e52a938ac0b7864f92f8910091c0da69201a156242df3b78
0bf505951741c44714527d252313b6959ce4f19d2e5512fca1c1b2da14424da3
0c82d00e6ee78b48559cda2f9cc909beeb8769183b115dfda0a5767832accc8d
2941bfd03b8933bb150b085a2252b69675495af64523bf8d38e67429e7cccb45
378a9b6862c8fa5c6915f158d16e4416243159bb9da44c564896c065bc6c1cf4
3ffcf5128fc404c2a363e3e8a8d4c8a7ae8c36fcacba7fdfe71ec9dabcadd567
40f5ca234e5eed1dc967c83fa99076ef636619148082f300cf877676728ebf14
42aad1ab6cc964043e53e5da13ed0f2b44a3bf6ae7702f60a805f13028377524
42bc8f509dbd6b9881cab4c1684d5cf74207046336f654db1b884197f15cae7b
47cf470c1c6de9af00b3b1ee963de8b94f51a2870b3338b3f33cfc565c0f8be4
587045d0ec69e47a3cc91b13c959aa80add9118ecfac47232ea992650f25f0b9
615e8851030f318751f3c8baf8fbfa9958e2dd7f25dc1a87dcf6d6f79d1f1a9f
63ec5e835e11203bbeef69095523344dd975f1ab52bdbf4a1db7a53914d967ca
714f9e16404b9ec83de56715e5387b2c4c2ed0af1889166a4e767822f971bf52
80ba9a145bf46cae605e911c18165c02213e8d11d68dc5b7824f259d17b7b6d0
86af714d79d18be1c9c0ae23cca9dbd7cef44530e253e80af5bd5c34eab09714
8a445cf5b6313cbe3b5872a55adde52aa8d1ae188f41d56f176e40a3137ac058
8e171f7367d1b68012ed1ceec8f54b7b9b8654ebaf63a760017c34d761b17878
8e98f35e65fb42c85eb4a2ab4793e294148e3f318252cb850a896274d2aa90bc
9d65ba6443863394a8c6582fef4a8aaab2fb46417eef41f1792cdbdb38ee0b4c
9da502ea8e7a768ee0dbafdc613db3df4a7cd9c98af08328265c4d2e953e8efa
9f2899688d2f23391cfd86e7b6d326a54f352bb294160878178639aab4aa378f
a2f3ccf973b3600c06c42dc3b867b263a788c18aa57f4448fea2525b7cbfd784
b2deaf9174086129ec3b9f79e05401fdb3baf8b75335addffac1950182d779df
b81870ebe27b98f6b8746349e8ea444c96bf2eaac5dbd6236175150ce579f46b
bd4dacd46031b2b837bc9bd06145b0571156fa496408ce728c003ae50b265aaf
c0775cfc822f59524b4ed714d257607fd5f2c9f0dc9f65763a86ffc33aac325b
c3596f717c495d20c33561e991d4295550b6d7544687f2363e999bdc0266224d
c54c4899c4d7dcda8b9e597aebfbaf7d65c9c7a760527d77e7fc9894283d736e
ca51ecf1633896f852929cb2d56ad1b5bed4ab6055bdcf370ced4011bed164aa
ce8b656cead478c34060510962daf97cea52abde68bbef7934dd5c5513cf6f3b
dafbb65569781083b627de833fb931cf98401299a62d747f03d8fc135ab57279
e193d395410520580e76a5b89b8d23a1d162c0e28c52cb8194d409a74a120f7d
e752efd93f9850ba0cafbbac01bb283c10095ac923cdb8ff027393001123d406
f728e5576d4ab63248c310396d67d9afa3267dd2dea3cfba690dbd04efe181fb
f75b6460b68d254f2195b08c606672fb55c05fb7bed7e16699b3231104b673ea
fe19f084021bdac5a9a5d270042ff53ef36357dd0743318d0480dee1a43de266

View File

@ -0,0 +1,34 @@
011a951a9796979c2b515ef4209662013bd1f16a20a1b35d1d950d7408bdc8b4
087e8d5f45f93a78e52a938ac0b7864f92f8910091c0da69201a156242df3b78
0bad18b7f2d82d7c9cf8e405262ad2f3dbe57928aa242c1070b917042a99072d
0bf505951741c44714527d252313b6959ce4f19d2e5512fca1c1b2da14424da3
0c82d00e6ee78b48559cda2f9cc909beeb8769183b115dfda0a5767832accc8d
2941bfd03b8933bb150b085a2252b69675495af64523bf8d38e67429e7cccb45
3ffcf5128fc404c2a363e3e8a8d4c8a7ae8c36fcacba7fdfe71ec9dabcadd567
40f5ca234e5eed1dc967c83fa99076ef636619148082f300cf877676728ebf14
42bc8f509dbd6b9881cab4c1684d5cf74207046336f654db1b884197f15cae7b
47cf470c1c6de9af00b3b1ee963de8b94f51a2870b3338b3f33cfc565c0f8be4
4b2e91022c34c756b7bd8ece046a2bab6f0dcad89f46c52d1f84cd48e8da55df
6416bc2321cdeb8758188af2b3925f2c82ffde014bf53b7a69c0f113a5c460fe
714f9e16404b9ec83de56715e5387b2c4c2ed0af1889166a4e767822f971bf52
80ba9a145bf46cae605e911c18165c02213e8d11d68dc5b7824f259d17b7b6d0
83bf0196cf45bbca0be7e292688a3622af7888c0e9ec01bb78edaff302cced06
8a445cf5b6313cbe3b5872a55adde52aa8d1ae188f41d56f176e40a3137ac058
8e98f35e65fb42c85eb4a2ab4793e294148e3f318252cb850a896274d2aa90bc
907acef01e05c3e0140858423e9284ddd3d64145ba8b0c3293371c5c7ab3d6b7
9d65ba6443863394a8c6582fef4a8aaab2fb46417eef41f1792cdbdb38ee0b4c
9da502ea8e7a768ee0dbafdc613db3df4a7cd9c98af08328265c4d2e953e8efa
a2f3ccf973b3600c06c42dc3b867b263a788c18aa57f4448fea2525b7cbfd784
b2deaf9174086129ec3b9f79e05401fdb3baf8b75335addffac1950182d779df
b3915971171e049292e28d7bc61fe362e94f73aa49b578f4ca1322b47d7fc39c
bd4dacd46031b2b837bc9bd06145b0571156fa496408ce728c003ae50b265aaf
c3596f717c495d20c33561e991d4295550b6d7544687f2363e999bdc0266224d
c54c4899c4d7dcda8b9e597aebfbaf7d65c9c7a760527d77e7fc9894283d736e
ca51ecf1633896f852929cb2d56ad1b5bed4ab6055bdcf370ced4011bed164aa
cb8001715217b4f6960aa24c1abb4b60a20c10f23abc1e5f69e0f5436bd788c8
d39c4c264e01ec47b0386da3775c6b0cc337974627ff55792938cca4895ac6c4
dafbb65569781083b627de833fb931cf98401299a62d747f03d8fc135ab57279
e193d395410520580e76a5b89b8d23a1d162c0e28c52cb8194d409a74a120f7d
e791912a7fad8954c764fae41d2958d2feeae2278e403429add9119ab43a36f5
f728e5576d4ab63248c310396d67d9afa3267dd2dea3cfba690dbd04efe181fb
fe19f084021bdac5a9a5d270042ff53ef36357dd0743318d0480dee1a43de266

View File

@ -0,0 +1,9 @@
35e13e123748cd27d1634c4e07e5ff2fc86901b09b215f3125331d1226c782be
378a9b6862c8fa5c6915f158d16e4416243159bb9da44c564896c065bc6c1cf4
42aad1ab6cc964043e53e5da13ed0f2b44a3bf6ae7702f60a805f13028377524
47cf470c1c6de9af00b3b1ee963de8b94f51a2870b3338b3f33cfc565c0f8be4
615e8851030f318751f3c8baf8fbfa9958e2dd7f25dc1a87dcf6d6f79d1f1a9f
83bf0196cf45bbca0be7e292688a3622af7888c0e9ec01bb78edaff302cced06
9d65ba6443863394a8c6582fef4a8aaab2fb46417eef41f1792cdbdb38ee0b4c
b3915971171e049292e28d7bc61fe362e94f73aa49b578f4ca1322b47d7fc39c
c0775cfc822f59524b4ed714d257607fd5f2c9f0dc9f65763a86ffc33aac325b