From 0c867b21ff8015b03076211980d73ec08857740f Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 6 Feb 2017 19:34:40 +0100 Subject: [PATCH 1/3] Add benchmark for checker --- src/restic/checker/checker_test.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/restic/checker/checker_test.go b/src/restic/checker/checker_test.go index d41f34b7b..40e6865a3 100644 --- a/src/restic/checker/checker_test.go +++ b/src/restic/checker/checker_test.go @@ -299,3 +299,28 @@ func TestCheckerModifiedData(t *testing.T) { t.Fatal("no error found, checker is broken") } } + +func BenchmarkChecker(t *testing.B) { + repodir, cleanup := test.Env(t, checkerTestData) + defer cleanup() + + repo := repository.TestOpenLocal(t, repodir) + + chkr := checker.New(repo) + hints, errs := chkr.LoadIndex() + if len(errs) > 0 { + t.Fatalf("expected no errors, got %v: %v", len(errs), errs) + } + + if len(hints) > 0 { + t.Errorf("expected no hints, got %v: %v", len(hints), hints) + } + + t.ResetTimer() + + for i := 0; i < t.N; i++ { + test.OKs(t, checkPacks(chkr)) + test.OKs(t, checkStruct(chkr)) + test.OKs(t, checkData(chkr)) + } +} From 436b5dc20cb734e13a3ac8f923ca748a5639d5bc Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 6 Feb 2017 19:50:27 +0100 Subject: [PATCH 2/3] Add Blob.String() --- src/restic/blob.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/restic/blob.go b/src/restic/blob.go index 670fd2fdf..35887121c 100644 --- a/src/restic/blob.go +++ b/src/restic/blob.go @@ -14,6 +14,11 @@ type Blob struct { Offset uint } +func (b Blob) String() string { + return fmt.Sprintf("", + b.Type, b.ID.Str(), b.Offset, b.Length) +} + // PackedBlob is a blob stored within a file. type PackedBlob struct { Blob From 1f81919d4a3b334e238a79facb576f295dfd5029 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 6 Feb 2017 21:17:52 +0100 Subject: [PATCH 3/3] checker: Reduce memory usage benchmark old bytes new bytes delta BenchmarkChecker-4 25551348 4288037 -83.22% --- src/restic/checker/checker.go | 64 +++++++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 10 deletions(-) diff --git a/src/restic/checker/checker.go b/src/restic/checker/checker.go index 7b37f7dd2..7a7170499 100644 --- a/src/restic/checker/checker.go +++ b/src/restic/checker/checker.go @@ -1,14 +1,17 @@ package checker import ( - "bytes" + "crypto/sha256" "fmt" + "io" + "io/ioutil" + "os" "sync" "restic/errors" + "restic/hashing" "restic" - "restic/backend" "restic/crypto" "restic/debug" "restic/pack" @@ -659,36 +662,77 @@ func (c *Checker) CountPacks() uint64 { func checkPack(r restic.Repository, id restic.ID) error { debug.Log("checking pack %v", id.Str()) h := restic.Handle{Type: restic.DataFile, Name: id.String()} - buf, err := backend.LoadAll(r.Backend(), h) + + rd, err := r.Backend().Load(h, 0, 0) if err != nil { return err } - hash := restic.Hash(buf) + packfile, err := ioutil.TempFile("", "restic-temp-check-") + if err != nil { + return errors.Wrap(err, "TempFile") + } + + defer func() { + packfile.Close() + os.Remove(packfile.Name()) + }() + + hrd := hashing.NewReader(rd, sha256.New()) + size, err := io.Copy(packfile, hrd) + if err != nil { + return errors.Wrap(err, "Copy") + } + + if err = rd.Close(); err != nil { + return err + } + + hash := restic.IDFromHash(hrd.Sum(nil)) + debug.Log("hash for pack %v is %v", id.Str(), hash.Str()) + if !hash.Equal(id) { debug.Log("Pack ID does not match, want %v, got %v", id.Str(), hash.Str()) return errors.Errorf("Pack ID does not match, want %v, got %v", id.Str(), hash.Str()) } - blobs, err := pack.List(r.Key(), bytes.NewReader(buf), int64(len(buf))) + blobs, err := pack.List(r.Key(), packfile, size) if err != nil { return err } var errs []error + var buf []byte for i, blob := range blobs { - debug.Log(" check blob %d: %v", i, blob.ID.Str()) + debug.Log(" check blob %d: %v", i, blob) - plainBuf := make([]byte, blob.Length) - n, err := crypto.Decrypt(r.Key(), plainBuf, buf[blob.Offset:blob.Offset+blob.Length]) + buf = buf[:cap(buf)] + if uint(len(buf)) < blob.Length { + buf = make([]byte, blob.Length) + } + buf = buf[:blob.Length] + + _, err := packfile.Seek(int64(blob.Offset), 0) + if err != nil { + return errors.Errorf("Seek(%v): %v", blob.Offset, err) + } + + _, err = io.ReadFull(packfile, buf) + if err != nil { + debug.Log(" error loading blob %v: %v", blob.ID.Str(), err) + errs = append(errs, errors.Errorf("blob %v: %v", i, err)) + continue + } + + n, err := crypto.Decrypt(r.Key(), buf, buf) if err != nil { debug.Log(" error decrypting blob %v: %v", blob.ID.Str(), err) errs = append(errs, errors.Errorf("blob %v: %v", i, err)) continue } - plainBuf = plainBuf[:n] + buf = buf[:n] - hash := restic.Hash(plainBuf) + hash := restic.Hash(buf) if !hash.Equal(blob.ID) { debug.Log(" Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str()) errs = append(errs, errors.Errorf("Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str()))