From f214dce87ca8313114fc9f613d6f9ed63e0f9bf9 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 18 Feb 2015 22:46:09 +0100 Subject: [PATCH] Add tests for Archiver.Preload() and a few more --- archiver_test.go | 52 ++++++++++++++++++++++++++++++++++------- server.go | 31 +++++++++++++++++++++++-- server_test.go | 60 +++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 132 insertions(+), 11 deletions(-) diff --git a/archiver_test.go b/archiver_test.go index 7689e274d..1285778d1 100644 --- a/archiver_test.go +++ b/archiver_test.go @@ -143,7 +143,7 @@ func BenchmarkArchiveDirectory(b *testing.B) { b.Logf("snapshot archived as %v", id) } -func snapshot(t *testing.T, server restic.Server, path string) *restic.Snapshot { +func snapshot(t testing.TB, server restic.Server, path string) *restic.Snapshot { arch, err := restic.NewArchiver(server, nil) ok(t, err) ok(t, arch.Preload()) @@ -152,7 +152,7 @@ func snapshot(t *testing.T, server restic.Server, path string) *restic.Snapshot return sn } -func countBlobs(t *testing.T, server restic.Server) int { +func countBlobs(t testing.TB, server restic.Server) int { blobs := 0 err := server.EachID(backend.Tree, func(id backend.ID) { tree, err := restic.LoadTree(server, id) @@ -165,7 +165,7 @@ func countBlobs(t *testing.T, server restic.Server) int { return blobs } -func TestArchiverPreload(t *testing.T) { +func archiveWithPreload(t testing.TB) { if *benchArchiveDirectory == "" { t.Skip("benchdir not set, skipping TestArchiverPreload") } @@ -177,7 +177,7 @@ func TestArchiverPreload(t *testing.T) { // archive a few files sn := snapshot(t, server, *benchArchiveDirectory) - t.Logf("archived snapshot %v", sn.ID) + t.Logf("archived snapshot %v", sn.ID()) // get archive stats blobsBefore := countBlobs(t, server) @@ -185,15 +185,51 @@ func TestArchiverPreload(t *testing.T) { // archive the same files again sn2 := snapshot(t, server, *benchArchiveDirectory) - t.Logf("archived snapshot %v", sn2.ID) + t.Logf("archived snapshot %v", sn2.ID()) // get archive stats blobsAfter := countBlobs(t, server) t.Logf("found %v blobs", blobsAfter) - // if there are more than 10% more blobs, something is wrong - if blobsAfter > (blobsBefore + blobsBefore/10) { + // if there are more than 50% more blobs, something is wrong + if blobsAfter > (blobsBefore + blobsBefore/2) { t.Fatalf("TestArchiverPreload: too many blobs in repository: before %d, after %d, threshhold %d", - blobsBefore, blobsAfter, (blobsBefore + blobsBefore/10)) + blobsBefore, blobsAfter, (blobsBefore + blobsBefore/2)) + } +} + +func TestArchivePreload(t *testing.T) { + archiveWithPreload(t) +} + +func BenchmarkArchivePreload(b *testing.B) { + archiveWithPreload(b) +} + +func BenchmarkPreload(t *testing.B) { + if *benchArchiveDirectory == "" { + t.Skip("benchdir not set, skipping TestArchiverPreload") + } + + be := setupBackend(t) + defer teardownBackend(t, be) + key := setupKey(t, be, "geheim") + server := restic.NewServerWithKey(be, key) + + // archive a few files + arch, err := restic.NewArchiver(server, nil) + ok(t, err) + sn, _, err := arch.Snapshot(*benchArchiveDirectory, nil) + ok(t, err) + t.Logf("archived snapshot %v", sn.ID()) + + // start benchmark + t.ResetTimer() + + for i := 0; i < t.N; i++ { + // create new archiver and preload + arch2, err := restic.NewArchiver(server, nil) + ok(t, err) + ok(t, arch2.Preload()) } } diff --git a/server.go b/server.go index 10e9afb7e..5c88fc1a4 100644 --- a/server.go +++ b/server.go @@ -1,6 +1,7 @@ package restic import ( + "bytes" "compress/zlib" "crypto/sha256" "encoding/json" @@ -107,6 +108,28 @@ func (s Server) LoadJSON(t backend.Type, blob Blob, item interface{}) error { return s.LoadJSONID(t, blob.Storage, item) } +var ( + zEmptyString = []byte("x\x9C\x03\x00\x00\x00\x00\x01") + zEmptyStringReader = bytes.NewReader(zEmptyString) +) + +var zReaderPool = sync.Pool{ + New: func() interface{} { + zEmptyStringReader.Seek(0, 0) + rd, err := zlib.NewReader(zEmptyStringReader) + if err != nil { + // shouldn't happen + panic(err) + } + return rd + }, +} + +type zReader interface { + io.ReadCloser + zlib.Resetter +} + // LoadJSONID calls Load() to get content from the backend and afterwards calls // json.Unmarshal on the item. func (s Server) LoadJSONID(t backend.Type, storageID backend.ID, item interface{}) error { @@ -125,8 +148,12 @@ func (s Server) LoadJSONID(t backend.Type, storageID backend.ID, item interface{ } // unzip - unzipRd, err := zlib.NewReader(decryptRd) - defer unzipRd.Close() + unzipRd := zReaderPool.Get().(zReader) + err = unzipRd.Reset(decryptRd, nil) + defer func() { + unzipRd.Close() + zReaderPool.Put(unzipRd) + }() if err != nil { return err } diff --git a/server_test.go b/server_test.go index c5c6f11d4..50fd9f391 100644 --- a/server_test.go +++ b/server_test.go @@ -136,9 +136,67 @@ func TestServerStats(t *testing.T) { // archive a few files sn := snapshot(t, server, *benchArchiveDirectory) - t.Logf("archived snapshot %v", sn.ID) + t.Logf("archived snapshot %v", sn.ID()) stats, err := server.Stats() ok(t, err) t.Logf("stats: %v", stats) } + +func TestLoadJSONID(t *testing.T) { + if *benchArchiveDirectory == "" { + t.Skip("benchdir not set, skipping TestServerStats") + } + + be := setupBackend(t) + defer teardownBackend(t, be) + key := setupKey(t, be, "geheim") + server := restic.NewServerWithKey(be, key) + + // archive a few files + sn := snapshot(t, server, *benchArchiveDirectory) + t.Logf("archived snapshot %v", sn.ID()) + + // benchmark loading first tree + list, err := server.List(backend.Tree) + ok(t, err) + assert(t, len(list) > 0, + "no Trees in repository found") + + treeID := list[0] + + tree := restic.NewTree() + err = server.LoadJSONID(backend.Tree, treeID, &tree) + ok(t, err) +} + +func BenchmarkLoadJSONID(t *testing.B) { + if *benchArchiveDirectory == "" { + t.Skip("benchdir not set, skipping TestServerStats") + } + + be := setupBackend(t) + defer teardownBackend(t, be) + key := setupKey(t, be, "geheim") + server := restic.NewServerWithKey(be, key) + + // archive a few files + sn := snapshot(t, server, *benchArchiveDirectory) + t.Logf("archived snapshot %v", sn.ID()) + + // benchmark loading first tree + list, err := server.List(backend.Tree) + ok(t, err) + assert(t, len(list) > 0, + "no Trees in repository found") + + t.ResetTimer() + + tree := restic.NewTree() + for i := 0; i < t.N; i++ { + for _, treeID := range list { + err = server.LoadJSONID(backend.Tree, treeID, &tree) + ok(t, err) + } + } +}