diff --git a/src/restic/archiver/archive_reader_test.go b/src/restic/archiver/archive_reader_test.go index b402bc6d1..e7e88d6cd 100644 --- a/src/restic/archiver/archive_reader_test.go +++ b/src/restic/archiver/archive_reader_test.go @@ -11,16 +11,16 @@ import ( "github.com/restic/chunker" ) -func loadBlob(t *testing.T, repo *repository.Repository, id restic.ID, buf []byte) []byte { - buf, err := repo.LoadBlob(id, restic.DataBlob, buf) +func loadBlob(t *testing.T, repo restic.Repository, id restic.ID, buf []byte) []byte { + n, err := repo.LoadDataBlob(id, buf) if err != nil { t.Fatalf("LoadBlob(%v) returned error %v", id, err) } - return buf + return buf[:n] } -func checkSavedFile(t *testing.T, repo *repository.Repository, treeID restic.ID, name string, rd io.Reader) { +func checkSavedFile(t *testing.T, repo restic.Repository, treeID restic.ID, name string, rd io.Reader) { tree, err := repo.LoadTree(treeID) if err != nil { t.Fatalf("LoadTree() returned error %v", err) diff --git a/src/restic/archiver/archiver_test.go b/src/restic/archiver/archiver_test.go index 1d43254cb..a1fa47683 100644 --- a/src/restic/archiver/archiver_test.go +++ b/src/restic/archiver/archiver_test.go @@ -12,8 +12,9 @@ import ( "restic/crypto" . "restic/test" - "github.com/restic/chunker" "restic/errors" + + "github.com/restic/chunker" ) var testPol = chunker.Pol(0x3DA3358B4DC173) @@ -126,6 +127,14 @@ func BenchmarkArchiveDirectory(b *testing.B) { } } +func countPacks(repo restic.Repository, t restic.FileType) (n uint) { + for _ = range repo.Backend().List(t, nil) { + n++ + } + + return n +} + func archiveWithDedup(t testing.TB) { repo := SetupRepo() defer TeardownRepo(repo) @@ -145,7 +154,7 @@ func archiveWithDedup(t testing.TB) { t.Logf("archived snapshot %v", sn.ID().Str()) // get archive stats - cnt.before.packs = repo.Count(restic.DataFile) + cnt.before.packs = countPacks(repo, restic.DataFile) cnt.before.dataBlobs = repo.Index().Count(restic.DataBlob) cnt.before.treeBlobs = repo.Index().Count(restic.TreeBlob) t.Logf("packs %v, data blobs %v, tree blobs %v", @@ -156,7 +165,7 @@ func archiveWithDedup(t testing.TB) { t.Logf("archived snapshot %v", sn2.ID().Str()) // get archive stats again - cnt.after.packs = repo.Count(restic.DataFile) + cnt.after.packs = countPacks(repo, restic.DataFile) cnt.after.dataBlobs = repo.Index().Count(restic.DataBlob) cnt.after.treeBlobs = repo.Index().Count(restic.TreeBlob) t.Logf("packs %v, data blobs %v, tree blobs %v", @@ -173,7 +182,7 @@ func archiveWithDedup(t testing.TB) { t.Logf("archived snapshot %v, parent %v", sn3.ID().Str(), sn2.ID().Str()) // get archive stats again - cnt.after2.packs = repo.Count(restic.DataFile) + cnt.after2.packs = countPacks(repo, restic.DataFile) cnt.after2.dataBlobs = repo.Index().Count(restic.DataBlob) cnt.after2.treeBlobs = repo.Index().Count(restic.TreeBlob) t.Logf("packs %v, data blobs %v, tree blobs %v", diff --git a/src/restic/checker/checker.go b/src/restic/checker/checker.go index df879fdfd..ebb416938 100644 --- a/src/restic/checker/checker.go +++ b/src/restic/checker/checker.go @@ -684,12 +684,13 @@ func checkPack(r restic.Repository, id restic.ID) error { debug.Log("Checker.checkPack", " check blob %d: %v", i, blob.ID.Str()) plainBuf := make([]byte, blob.Length) - plainBuf, err = crypto.Decrypt(r.Key(), plainBuf, buf[blob.Offset:blob.Offset+blob.Length]) + n, err := crypto.Decrypt(r.Key(), plainBuf, buf[blob.Offset:blob.Offset+blob.Length]) if err != nil { debug.Log("Checker.checkPack", " error decrypting blob %v: %v", blob.ID.Str(), err) errs = append(errs, errors.Errorf("blob %v: %v", i, err)) continue } + plainBuf = plainBuf[:n] hash := restic.Hash(plainBuf) if !hash.Equal(blob.ID) { diff --git a/src/restic/checker/checker_test.go b/src/restic/checker/checker_test.go index 6e9f29d06..0037f0adb 100644 --- a/src/restic/checker/checker_test.go +++ b/src/restic/checker/checker_test.go @@ -17,7 +17,7 @@ import ( var checkerTestData = filepath.Join("testdata", "checker-test-repo.tar.gz") -func list(repo *repository.Repository, t restic.FileType) (IDs []string) { +func list(repo restic.Repository, t restic.FileType) (IDs []string) { done := make(chan struct{}) defer close(done) diff --git a/src/restic/checker/testing.go b/src/restic/checker/testing.go index 3bf9aa2ec..7b642dea1 100644 --- a/src/restic/checker/testing.go +++ b/src/restic/checker/testing.go @@ -1,12 +1,12 @@ package checker import ( - "restic/repository" + "restic" "testing" ) // TestCheckRepo runs the checker on repo. -func TestCheckRepo(t testing.TB, repo *repository.Repository) { +func TestCheckRepo(t testing.TB, repo restic.Repository) { chkr := New(repo) hints, errs := chkr.LoadIndex() diff --git a/src/restic/crypto/crypto.go b/src/restic/crypto/crypto.go index 2ebf5d31b..57fdd6230 100644 --- a/src/restic/crypto/crypto.go +++ b/src/restic/crypto/crypto.go @@ -274,9 +274,9 @@ func Encrypt(ks *Key, ciphertext []byte, plaintext []byte) ([]byte, error) { // Decrypt verifies and decrypts the ciphertext. Ciphertext must be in the form // IV || Ciphertext || MAC. plaintext and ciphertext may point to (exactly) the // same slice. -func Decrypt(ks *Key, plaintext []byte, ciphertextWithMac []byte) ([]byte, error) { +func Decrypt(ks *Key, plaintext []byte, ciphertextWithMac []byte) (int, error) { if !ks.Valid() { - return nil, errors.New("invalid key") + return 0, errors.New("invalid key") } // check for plausible length @@ -284,21 +284,26 @@ func Decrypt(ks *Key, plaintext []byte, ciphertextWithMac []byte) ([]byte, error panic("trying to decrypt invalid data: ciphertext too small") } + // check buffer length for plaintext + plaintextLength := len(ciphertextWithMac) - ivSize - macSize + if len(plaintext) < plaintextLength { + return 0, errors.Errorf("plaintext buffer too small, %d < %d", len(plaintext), plaintextLength) + } + // extract mac l := len(ciphertextWithMac) - macSize ciphertextWithIV, mac := ciphertextWithMac[:l], ciphertextWithMac[l:] // verify mac if !poly1305Verify(ciphertextWithIV[ivSize:], ciphertextWithIV[:ivSize], &ks.MAC, mac) { - return nil, ErrUnauthenticated + return 0, ErrUnauthenticated } // extract iv iv, ciphertext := ciphertextWithIV[:ivSize], ciphertextWithIV[ivSize:] - if cap(plaintext) < len(ciphertext) { - // extend plaintext - plaintext = append(plaintext, make([]byte, len(ciphertext)-cap(plaintext))...) + if len(ciphertext) != plaintextLength { + return 0, errors.Errorf("plaintext and ciphertext lengths do not match: %d != %d", len(ciphertext), plaintextLength) } // decrypt data @@ -312,7 +317,7 @@ func Decrypt(ks *Key, plaintext []byte, ciphertextWithMac []byte) ([]byte, error plaintext = plaintext[:len(ciphertext)] e.XORKeyStream(plaintext, ciphertext) - return plaintext, nil + return plaintextLength, nil } // Valid tests if the key is valid. diff --git a/src/restic/crypto/crypto_int_test.go b/src/restic/crypto/crypto_int_test.go index 5fed6b54c..1dbc32623 100644 --- a/src/restic/crypto/crypto_int_test.go +++ b/src/restic/crypto/crypto_int_test.go @@ -100,15 +100,17 @@ func TestCrypto(t *testing.T) { } // decrypt message - _, err = Decrypt(k, []byte{}, msg) + buf := make([]byte, len(tv.plaintext)) + n, err := Decrypt(k, buf, msg) if err != nil { t.Fatal(err) } + buf = buf[:n] // change mac, this must fail msg[len(msg)-8] ^= 0x23 - if _, err = Decrypt(k, []byte{}, msg); err != ErrUnauthenticated { + if _, err = Decrypt(k, buf, msg); err != ErrUnauthenticated { t.Fatal("wrong MAC value not detected") } @@ -118,15 +120,17 @@ func TestCrypto(t *testing.T) { // tamper with message, this must fail msg[16+5] ^= 0x85 - if _, err = Decrypt(k, []byte{}, msg); err != ErrUnauthenticated { + if _, err = Decrypt(k, buf, msg); err != ErrUnauthenticated { t.Fatal("tampered message not detected") } // test decryption - p, err := Decrypt(k, []byte{}, tv.ciphertext) + p := make([]byte, len(tv.ciphertext)) + n, err = Decrypt(k, p, tv.ciphertext) if err != nil { t.Fatal(err) } + p = p[:n] if !bytes.Equal(p, tv.plaintext) { t.Fatalf("wrong plaintext: expected %q but got %q\n", tv.plaintext, p) diff --git a/src/restic/crypto/crypto_test.go b/src/restic/crypto/crypto_test.go index fe799da77..39c3cc169 100644 --- a/src/restic/crypto/crypto_test.go +++ b/src/restic/crypto/crypto_test.go @@ -32,8 +32,10 @@ func TestEncryptDecrypt(t *testing.T) { "ciphertext length does not match: want %d, got %d", len(data)+crypto.Extension, len(ciphertext)) - plaintext, err := crypto.Decrypt(k, nil, ciphertext) + plaintext := make([]byte, len(ciphertext)) + n, err := crypto.Decrypt(k, plaintext, ciphertext) OK(t, err) + plaintext = plaintext[:n] Assert(t, len(plaintext) == len(data), "plaintext length does not match: want %d, got %d", len(data), len(plaintext)) @@ -58,8 +60,10 @@ func TestSmallBuffer(t *testing.T) { cap(ciphertext)) // check for the correct plaintext - plaintext, err := crypto.Decrypt(k, nil, ciphertext) + plaintext := make([]byte, len(ciphertext)) + n, err := crypto.Decrypt(k, plaintext, ciphertext) OK(t, err) + plaintext = plaintext[:n] Assert(t, bytes.Equal(plaintext, data), "wrong plaintext returned") } @@ -78,8 +82,9 @@ func TestSameBuffer(t *testing.T) { OK(t, err) // use the same buffer for decryption - ciphertext, err = crypto.Decrypt(k, ciphertext, ciphertext) + n, err := crypto.Decrypt(k, ciphertext, ciphertext) OK(t, err) + ciphertext = ciphertext[:n] Assert(t, bytes.Equal(ciphertext, data), "wrong plaintext returned") } @@ -97,9 +102,9 @@ func TestCornerCases(t *testing.T) { len(c)) // this should decrypt to nil - p, err := crypto.Decrypt(k, nil, c) + n, err := crypto.Decrypt(k, nil, c) OK(t, err) - Equals(t, []byte(nil), p) + Equals(t, 0, n) // test encryption for same slice, this should return an error _, err = crypto.Encrypt(k, c, c) @@ -160,7 +165,7 @@ func BenchmarkDecrypt(b *testing.B) { b.SetBytes(int64(size)) for i := 0; i < b.N; i++ { - plaintext, err = crypto.Decrypt(k, plaintext, ciphertext) + _, err = crypto.Decrypt(k, plaintext, ciphertext) OK(b, err) } } diff --git a/src/restic/fuse/dir.go b/src/restic/fuse/dir.go index 14f8c7f21..004d02086 100644 --- a/src/restic/fuse/dir.go +++ b/src/restic/fuse/dir.go @@ -12,7 +12,6 @@ import ( "restic" "restic/debug" - "restic/repository" ) // Statically ensure that *dir implement those interface @@ -20,14 +19,14 @@ var _ = fs.HandleReadDirAller(&dir{}) var _ = fs.NodeStringLookuper(&dir{}) type dir struct { - repo *repository.Repository + repo restic.Repository items map[string]*restic.Node inode uint64 node *restic.Node ownerIsRoot bool } -func newDir(repo *repository.Repository, node *restic.Node, ownerIsRoot bool) (*dir, error) { +func newDir(repo restic.Repository, node *restic.Node, ownerIsRoot bool) (*dir, error) { debug.Log("newDir", "new dir for %v (%v)", node.Name, node.Subtree.Str()) tree, err := repo.LoadTree(*node.Subtree) if err != nil { @@ -50,7 +49,7 @@ func newDir(repo *repository.Repository, node *restic.Node, ownerIsRoot bool) (* // replaceSpecialNodes replaces nodes with name "." and "/" by their contents. // Otherwise, the node is returned. -func replaceSpecialNodes(repo *repository.Repository, node *restic.Node) ([]*restic.Node, error) { +func replaceSpecialNodes(repo restic.Repository, node *restic.Node) ([]*restic.Node, error) { if node.Type != "dir" || node.Subtree == nil { return []*restic.Node{node}, nil } @@ -67,7 +66,7 @@ func replaceSpecialNodes(repo *repository.Repository, node *restic.Node) ([]*res return tree.Nodes, nil } -func newDirFromSnapshot(repo *repository.Repository, snapshot SnapshotWithId, ownerIsRoot bool) (*dir, error) { +func newDirFromSnapshot(repo restic.Repository, snapshot SnapshotWithId, ownerIsRoot bool) (*dir, error) { debug.Log("newDirFromSnapshot", "new dir for snapshot %v (%v)", snapshot.ID.Str(), snapshot.Tree.Str()) tree, err := repo.LoadTree(*snapshot.Tree) if err != nil { diff --git a/src/restic/fuse/file.go b/src/restic/fuse/file.go index d5949ee86..ae1b90124 100644 --- a/src/restic/fuse/file.go +++ b/src/restic/fuse/file.go @@ -27,7 +27,7 @@ var _ = fs.HandleReleaser(&file{}) // for fuse operations. type BlobLoader interface { LookupBlobSize(restic.ID, restic.BlobType) (uint, error) - LoadBlob(restic.ID, restic.BlobType, []byte) ([]byte, error) + LoadDataBlob(restic.ID, []byte) (int, error) } type file struct { @@ -109,12 +109,12 @@ func (f *file) getBlobAt(i int) (blob []byte, err error) { buf = make([]byte, f.sizes[i]) } - blob, err = f.repo.LoadBlob(f.node.Content[i], restic.DataBlob, buf) + n, err := f.repo.LoadDataBlob(f.node.Content[i], buf) if err != nil { debug.Log("file.getBlobAt", "LoadBlob(%v, %v) failed: %v", f.node.Name, f.node.Content[i], err) return nil, err } - f.blobs[i] = blob + f.blobs[i] = buf[:n] return blob, nil } diff --git a/src/restic/fuse/file_test.go b/src/restic/fuse/file_test.go index 58e6b33ba..0101cadc9 100644 --- a/src/restic/fuse/file_test.go +++ b/src/restic/fuse/file_test.go @@ -34,19 +34,19 @@ func (m *MockRepo) LookupBlobSize(id restic.ID, t restic.BlobType) (uint, error) return uint(len(buf)), nil } -func (m *MockRepo) LoadBlob(id restic.ID, t restic.BlobType, buf []byte) ([]byte, error) { - size, err := m.LookupBlobSize(id, t) +func (m *MockRepo) LoadDataBlob(id restic.ID, buf []byte) (int, error) { + size, err := m.LookupBlobSize(id, restic.DataBlob) if err != nil { - return nil, err + return 0, err } if uint(cap(buf)) < size { - return nil, errors.New("buffer too small") + return 0, errors.New("buffer too small") } buf = buf[:size] copy(buf, m.blobs[id]) - return buf, nil + return int(size), nil } type MockContext struct{} diff --git a/src/restic/fuse/link.go b/src/restic/fuse/link.go index 732446a7a..43fb35020 100644 --- a/src/restic/fuse/link.go +++ b/src/restic/fuse/link.go @@ -5,7 +5,6 @@ package fuse import ( "restic" - "restic/repository" "bazil.org/fuse" "bazil.org/fuse/fs" @@ -20,7 +19,7 @@ type link struct { ownerIsRoot bool } -func newLink(repo *repository.Repository, node *restic.Node, ownerIsRoot bool) (*link, error) { +func newLink(repo restic.Repository, node *restic.Node, ownerIsRoot bool) (*link, error) { return &link{node: node, ownerIsRoot: ownerIsRoot}, nil } diff --git a/src/restic/fuse/snapshot.go b/src/restic/fuse/snapshot.go index 8d14823b0..b97e3ced9 100644 --- a/src/restic/fuse/snapshot.go +++ b/src/restic/fuse/snapshot.go @@ -13,7 +13,6 @@ import ( "restic" "restic/debug" - "restic/repository" "golang.org/x/net/context" ) @@ -30,7 +29,7 @@ var _ = fs.HandleReadDirAller(&SnapshotsDir{}) var _ = fs.NodeStringLookuper(&SnapshotsDir{}) type SnapshotsDir struct { - repo *repository.Repository + repo restic.Repository ownerIsRoot bool // knownSnapshots maps snapshot timestamp to the snapshot @@ -38,7 +37,7 @@ type SnapshotsDir struct { knownSnapshots map[string]SnapshotWithId } -func NewSnapshotsDir(repo *repository.Repository, ownerIsRoot bool) *SnapshotsDir { +func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool) *SnapshotsDir { debug.Log("NewSnapshotsDir", "fuse mount initiated") return &SnapshotsDir{ repo: repo, diff --git a/src/restic/index/index_test.go b/src/restic/index/index_test.go index 521d0c0b0..f1378531f 100644 --- a/src/restic/index/index_test.go +++ b/src/restic/index/index_test.go @@ -15,7 +15,7 @@ var ( depth = 3 ) -func createFilledRepo(t testing.TB, snapshots int, dup float32) (*repository.Repository, func()) { +func createFilledRepo(t testing.TB, snapshots int, dup float32) (restic.Repository, func()) { repo, cleanup := repository.TestRepository(t) for i := 0; i < 3; i++ { @@ -25,7 +25,7 @@ func createFilledRepo(t testing.TB, snapshots int, dup float32) (*repository.Rep return repo, cleanup } -func validateIndex(t testing.TB, repo *repository.Repository, idx *Index) { +func validateIndex(t testing.TB, repo restic.Repository, idx *Index) { for id := range repo.List(restic.DataFile, nil) { if _, ok := idx.Packs[id]; !ok { t.Errorf("pack %v missing from index", id.Str()) @@ -162,7 +162,7 @@ func TestIndexDuplicateBlobs(t *testing.T) { t.Logf("%d packs with duplicate blobs", len(packs)) } -func loadIndex(t testing.TB, repo *repository.Repository) *Index { +func loadIndex(t testing.TB, repo restic.Repository) *Index { idx, err := Load(repo, nil) if err != nil { t.Fatalf("Load() returned error %v", err) diff --git a/src/restic/pack/pack.go b/src/restic/pack/pack.go index 40d10839b..17f79b09a 100644 --- a/src/restic/pack/pack.go +++ b/src/restic/pack/pack.go @@ -225,12 +225,13 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries []restic.Blob, err return nil, err } - hdr, err := crypto.Decrypt(k, buf, buf) + n, err := crypto.Decrypt(k, buf, buf) if err != nil { return nil, err } + buf = buf[:n] - hdrRd := bytes.NewReader(hdr) + hdrRd := bytes.NewReader(buf) pos := uint(0) for { diff --git a/src/restic/repository.go b/src/restic/repository.go index 5c1067f7b..c7e8ae170 100644 --- a/src/restic/repository.go +++ b/src/restic/repository.go @@ -16,6 +16,7 @@ type Repository interface { Index() Index SaveFullIndex() error SaveIndex() error + LoadIndex() error Config() Config diff --git a/src/restic/repository/key.go b/src/restic/repository/key.go index b874dc644..9a0a85c21 100644 --- a/src/restic/repository/key.go +++ b/src/restic/repository/key.go @@ -85,10 +85,12 @@ func OpenKey(s *Repository, name string, password string) (*Key, error) { } // decrypt master keys - buf, err := crypto.Decrypt(k.user, []byte{}, k.Data) + buf := make([]byte, len(k.Data)) + n, err := crypto.Decrypt(k.user, buf, k.Data) if err != nil { return nil, err } + buf = buf[:n] // restore json k.master = &crypto.Key{} diff --git a/src/restic/repository/repack.go b/src/restic/repository/repack.go index 274f2d320..2ce701ad9 100644 --- a/src/restic/repository/repack.go +++ b/src/restic/repository/repack.go @@ -15,7 +15,7 @@ import ( // these packs. Each pack is loaded and the blobs listed in keepBlobs is saved // into a new pack. Afterwards, the packs are removed. This operation requires // an exclusive lock on the repo. -func Repack(repo *Repository, packs restic.IDSet, keepBlobs restic.BlobSet) (err error) { +func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet) (err error) { debug.Log("Repack", "repacking %d packs while keeping %d blobs", len(packs), len(keepBlobs)) buf := make([]byte, 0, maxPackSize) @@ -48,16 +48,21 @@ func Repack(repo *Repository, packs restic.IDSet, keepBlobs restic.BlobSet) (err continue } - ciphertext := buf[entry.Offset : entry.Offset+entry.Length] + debug.Log("Repack", " process blob %v", h) - if cap(plaintext) < len(ciphertext) { + ciphertext := buf[entry.Offset : entry.Offset+entry.Length] + plaintext = plaintext[:len(plaintext)] + if len(plaintext) < len(ciphertext) { plaintext = make([]byte, len(ciphertext)) } - plaintext, err = crypto.Decrypt(repo.Key(), plaintext, ciphertext) + debug.Log("Repack", " ciphertext %d, plaintext %d", len(plaintext), len(ciphertext)) + + n, err := crypto.Decrypt(repo.Key(), plaintext, ciphertext) if err != nil { return err } + plaintext = plaintext[:n] _, err = repo.SaveAndEncrypt(entry.Type, plaintext, &entry.ID) if err != nil { diff --git a/src/restic/repository/repack_test.go b/src/restic/repository/repack_test.go index 026e43cbc..9b118d7b4 100644 --- a/src/restic/repository/repack_test.go +++ b/src/restic/repository/repack_test.go @@ -23,7 +23,7 @@ func random(t testing.TB, length int) []byte { return buf } -func createRandomBlobs(t testing.TB, repo *repository.Repository, blobs int, pData float32) { +func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData float32) { for i := 0; i < blobs; i++ { var ( tpe restic.BlobType @@ -65,7 +65,7 @@ func createRandomBlobs(t testing.TB, repo *repository.Repository, blobs int, pDa // selectBlobs splits the list of all blobs randomly into two lists. A blob // will be contained in the firstone ith probability p. -func selectBlobs(t *testing.T, repo *repository.Repository, p float32) (list1, list2 restic.BlobSet) { +func selectBlobs(t *testing.T, repo restic.Repository, p float32) (list1, list2 restic.BlobSet) { done := make(chan struct{}) defer close(done) @@ -100,7 +100,7 @@ func selectBlobs(t *testing.T, repo *repository.Repository, p float32) (list1, l return list1, list2 } -func listPacks(t *testing.T, repo *repository.Repository) restic.IDSet { +func listPacks(t *testing.T, repo restic.Repository) restic.IDSet { done := make(chan struct{}) defer close(done) @@ -112,7 +112,7 @@ func listPacks(t *testing.T, repo *repository.Repository) restic.IDSet { return list } -func findPacksForBlobs(t *testing.T, repo *repository.Repository, blobs restic.BlobSet) restic.IDSet { +func findPacksForBlobs(t *testing.T, repo restic.Repository, blobs restic.BlobSet) restic.IDSet { packs := restic.NewIDSet() idx := repo.Index() @@ -130,26 +130,26 @@ func findPacksForBlobs(t *testing.T, repo *repository.Repository, blobs restic.B return packs } -func repack(t *testing.T, repo *repository.Repository, packs restic.IDSet, blobs restic.BlobSet) { +func repack(t *testing.T, repo restic.Repository, packs restic.IDSet, blobs restic.BlobSet) { err := repository.Repack(repo, packs, blobs) if err != nil { t.Fatal(err) } } -func saveIndex(t *testing.T, repo *repository.Repository) { +func saveIndex(t *testing.T, repo restic.Repository) { if err := repo.SaveIndex(); err != nil { t.Fatalf("repo.SaveIndex() %v", err) } } -func rebuildIndex(t *testing.T, repo *repository.Repository) { +func rebuildIndex(t *testing.T, repo restic.Repository) { if err := repository.RebuildIndex(repo); err != nil { t.Fatalf("error rebuilding index: %v", err) } } -func reloadIndex(t *testing.T, repo *repository.Repository) { +func reloadIndex(t *testing.T, repo restic.Repository) { repo.SetIndex(repository.NewMasterIndex()) if err := repo.LoadIndex(); err != nil { t.Fatalf("error loading new index: %v", err) diff --git a/src/restic/repository/repository.go b/src/restic/repository/repository.go index 87025425d..e0d3659c2 100644 --- a/src/restic/repository/repository.go +++ b/src/restic/repository/repository.go @@ -72,20 +72,22 @@ func (r *Repository) LoadAndDecrypt(t restic.FileType, id restic.ID) ([]byte, er return nil, errors.New("invalid data returned") } + plain := make([]byte, len(buf)) + // decrypt - plain, err := r.Decrypt(buf) + n, err := r.decryptTo(plain, buf) if err != nil { return nil, err } - return plain, nil + return plain[:n], nil } // LoadBlob tries to load and decrypt content identified by t and id from a // pack from the backend, the result is stored in plaintextBuf, which must be // large enough to hold the complete blob. func (r *Repository) LoadBlob(id restic.ID, t restic.BlobType, plaintextBuf []byte) ([]byte, error) { - debug.Log("Repo.LoadBlob", "load %v with id %v", t, id.Str()) + debug.Log("Repo.LoadBlob", "load %v with id %v (buf %d)", t, id.Str(), len(plaintextBuf)) // lookup plaintext size of blob size, err := r.idx.LookupSize(id, t) @@ -94,11 +96,8 @@ func (r *Repository) LoadBlob(id restic.ID, t restic.BlobType, plaintextBuf []by } // make sure the plaintext buffer is large enough, extend otherwise - plaintextBufSize := uint(cap(plaintextBuf)) - if size > plaintextBufSize { - debug.Log("Repo.LoadBlob", "need to expand buffer: want %d bytes, got %d", - size, plaintextBufSize) - plaintextBuf = make([]byte, size) + if len(plaintextBuf) < int(size) { + return nil, errors.Errorf("buffer is too small: %d < %d", len(plaintextBuf), size) } // lookup packs @@ -134,11 +133,12 @@ func (r *Repository) LoadBlob(id restic.ID, t restic.BlobType, plaintextBuf []by } // decrypt - plaintextBuf, err = r.decryptTo(plaintextBuf, ciphertextBuf) + n, err = r.decryptTo(plaintextBuf, ciphertextBuf) if err != nil { lastError = errors.Errorf("decrypting blob %v failed: %v", id, err) continue } + plaintextBuf = plaintextBuf[:n] // check hash if !restic.Hash(plaintextBuf).Equal(id) { @@ -403,7 +403,7 @@ func (r *Repository) LoadIndex() error { } // LoadIndex loads the index id from backend and returns it. -func LoadIndex(repo *Repository, id restic.ID) (*Index, error) { +func LoadIndex(repo restic.Repository, id restic.ID) (*Index, error) { idx, err := LoadIndexWithDecoder(repo, id, DecodeIndex) if err == nil { return idx, nil @@ -467,19 +467,14 @@ func (r *Repository) init(password string, cfg restic.Config) error { return err } -// Decrypt authenticates and decrypts ciphertext and returns the plaintext. -func (r *Repository) Decrypt(ciphertext []byte) ([]byte, error) { - return r.decryptTo(nil, ciphertext) -} - // decrypt authenticates and decrypts ciphertext and stores the result in // plaintext. -func (r *Repository) decryptTo(plaintext, ciphertext []byte) ([]byte, error) { +func (r *Repository) decryptTo(plaintext, ciphertext []byte) (int, error) { if r.key == nil { - return nil, errors.New("key for repository not set") + return 0, errors.New("key for repository not set") } - return crypto.Decrypt(r.key, nil, ciphertext) + return crypto.Decrypt(r.key, plaintext, ciphertext) } // Encrypt encrypts and authenticates the plaintext and saves the result in @@ -502,15 +497,6 @@ func (r *Repository) KeyName() string { return r.keyName } -// Count returns the number of blobs of a given type in the backend. -func (r *Repository) Count(t restic.FileType) (n uint) { - for _ = range r.be.List(t, nil) { - n++ - } - - return -} - func (r *Repository) list(t restic.FileType, done <-chan struct{}, out chan<- restic.ID) { defer close(out) in := r.be.List(t, done) @@ -592,14 +578,17 @@ func (r *Repository) Close() error { // LoadTree loads a tree from the repository. func (r *Repository) LoadTree(id restic.ID) (*restic.Tree, error) { + debug.Log("repo.LoadTree", "load tree %v", id.Str()) + size, err := r.idx.LookupSize(id, restic.TreeBlob) if err != nil { return nil, err } + debug.Log("repo.LoadTree", "size is %d, create buffer", size) buf := make([]byte, size) - buf, err = r.LoadBlob(id, restic.TreeBlob, nil) + buf, err = r.LoadBlob(id, restic.TreeBlob, buf) if err != nil { return nil, err } @@ -615,6 +604,7 @@ func (r *Repository) LoadTree(id restic.ID) (*restic.Tree, error) { // LoadDataBlob loads a data blob from the repository to the buffer. func (r *Repository) LoadDataBlob(id restic.ID, buf []byte) (int, error) { + debug.Log("repo.LoadDataBlob", "load blob %v into buf %p", id.Str(), buf) size, err := r.idx.LookupSize(id, restic.DataBlob) if err != nil { return 0, err @@ -629,5 +619,7 @@ func (r *Repository) LoadDataBlob(id restic.ID, buf []byte) (int, error) { return 0, err } + debug.Log("repo.LoadDataBlob", "loaded %d bytes into buf %p", len(buf), buf) + return len(buf), err } diff --git a/src/restic/repository/repository_test.go b/src/restic/repository/repository_test.go index 98b8edd84..644650c30 100644 --- a/src/restic/repository/repository_test.go +++ b/src/restic/repository/repository_test.go @@ -90,8 +90,10 @@ func TestSave(t *testing.T) { // OK(t, repo.SaveIndex()) // read back - buf, err := repo.LoadBlob(id, restic.DataBlob, make([]byte, size)) + buf := make([]byte, size) + n, err := repo.LoadDataBlob(id, buf) OK(t, err) + Equals(t, len(buf), n) Assert(t, len(buf) == len(data), "number of bytes read back does not match: expected %d, got %d", @@ -122,8 +124,10 @@ func TestSaveFrom(t *testing.T) { OK(t, repo.Flush()) // read back - buf, err := repo.LoadBlob(id, restic.DataBlob, make([]byte, size)) + buf := make([]byte, size) + n, err := repo.LoadDataBlob(id, buf) OK(t, err) + Equals(t, len(buf), n) Assert(t, len(buf) == len(data), "number of bytes read back does not match: expected %d, got %d", @@ -157,7 +161,7 @@ func BenchmarkSaveAndEncrypt(t *testing.B) { } } -func TestLoadJSONPack(t *testing.T) { +func TestLoadTree(t *testing.T) { repo := SetupRepo() defer TeardownRepo(repo) @@ -169,12 +173,11 @@ func TestLoadJSONPack(t *testing.T) { sn := SnapshotDir(t, repo, BenchArchiveDirectory, nil) OK(t, repo.Flush()) - tree := restic.NewTree() - err := repo.LoadJSONPack(restic.TreeBlob, *sn.Tree, &tree) + _, err := repo.LoadTree(*sn.Tree) OK(t, err) } -func BenchmarkLoadJSONPack(t *testing.B) { +func BenchmarkLoadTree(t *testing.B) { repo := SetupRepo() defer TeardownRepo(repo) @@ -186,12 +189,10 @@ func BenchmarkLoadJSONPack(t *testing.B) { sn := SnapshotDir(t, repo, BenchArchiveDirectory, nil) OK(t, repo.Flush()) - tree := restic.NewTree() - t.ResetTimer() for i := 0; i < t.N; i++ { - err := repo.LoadJSONPack(restic.TreeBlob, *sn.Tree, &tree) + _, err := repo.LoadTree(*sn.Tree) OK(t, err) } } @@ -244,7 +245,7 @@ func BenchmarkLoadIndex(b *testing.B) { } // saveRandomDataBlobs generates random data blobs and saves them to the repository. -func saveRandomDataBlobs(t testing.TB, repo *repository.Repository, num int, sizeMax int) { +func saveRandomDataBlobs(t testing.TB, repo restic.Repository, num int, sizeMax int) { for i := 0; i < num; i++ { size := mrand.Int() % sizeMax diff --git a/src/restic/test/backend.go b/src/restic/test/backend.go index d73f0d4fe..e8710b8c8 100644 --- a/src/restic/test/backend.go +++ b/src/restic/test/backend.go @@ -49,7 +49,7 @@ func getBoolVar(name string, defaultValue bool) bool { return defaultValue } -func SetupRepo() *repository.Repository { +func SetupRepo() restic.Repository { tempdir, err := ioutil.TempDir(TestTempDir, "restic-test-") if err != nil { panic(err) @@ -70,27 +70,29 @@ func SetupRepo() *repository.Repository { return repo } -func TeardownRepo(repo *repository.Repository) { +func TeardownRepo(repo restic.Repository) { if !TestCleanupTempDirs { l := repo.Backend().(*local.Local) fmt.Printf("leaving local backend at %s\n", l.Location()) return } - err := repo.Delete() - if err != nil { - panic(err) + if r, ok := repo.(restic.Deleter); ok { + err := r.Delete() + if err != nil { + panic(err) + } } } -func SnapshotDir(t testing.TB, repo *repository.Repository, path string, parent *restic.ID) *restic.Snapshot { +func SnapshotDir(t testing.TB, repo restic.Repository, path string, parent *restic.ID) *restic.Snapshot { arch := archiver.New(repo) sn, _, err := arch.Snapshot(nil, []string{path}, parent) OK(t, err) return sn } -func WithRepo(t testing.TB, f func(*repository.Repository)) { +func WithRepo(t testing.TB, f func(restic.Repository)) { repo := SetupRepo() f(repo) TeardownRepo(repo) diff --git a/src/restic/test/helpers.go b/src/restic/test/helpers.go index 6c7ee8de1..2fbdb83d6 100644 --- a/src/restic/test/helpers.go +++ b/src/restic/test/helpers.go @@ -34,7 +34,7 @@ func Assert(tb testing.TB, condition bool, msg string, v ...interface{}) { func OK(tb testing.TB, err error) { if err != nil { _, file, line, _ := runtime.Caller(1) - fmt.Printf("\033[31m%s:%d: unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error()) + fmt.Printf("\033[31m%s:%d: unexpected error: %+v\033[39m\n\n", filepath.Base(file), line, err) tb.FailNow() } } @@ -209,7 +209,7 @@ func WithTestEnvironment(t testing.TB, repoFixture string, f func(repodir string } // OpenLocalRepo opens the local repository located at dir. -func OpenLocalRepo(t testing.TB, dir string) *repository.Repository { +func OpenLocalRepo(t testing.TB, dir string) restic.Repository { be, err := local.Open(dir) OK(t, err) diff --git a/src/restic/testing.go b/src/restic/testing.go index e4fe6ddb3..9b6b627c8 100644 --- a/src/restic/testing.go +++ b/src/restic/testing.go @@ -8,8 +8,9 @@ import ( "testing" "time" - "github.com/restic/chunker" "restic/errors" + + "github.com/restic/chunker" ) // fakeFile returns a reader which yields deterministic pseudo-random data. diff --git a/src/restic/walk/walk_test.go b/src/restic/walk/walk_test.go index 221f5df9c..681c2f1e6 100644 --- a/src/restic/walk/walk_test.go +++ b/src/restic/walk/walk_test.go @@ -10,7 +10,6 @@ import ( "restic" "restic/archiver" "restic/pipe" - "restic/repository" . "restic/test" "restic/walk" ) @@ -91,7 +90,7 @@ func TestWalkTree(t *testing.T) { } type delayRepo struct { - repo *repository.Repository + repo restic.Repository delay time.Duration }