From db0e3cd7724c2836239257b1db7b7e35da3675c2 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 16 Jul 2017 20:02:59 +0200 Subject: [PATCH 1/2] repo: Remove packer limits This commit simplifies finding a packer: The first open packer is taken, and the upper limit for the pack file is removed. --- internal/repository/packer_manager.go | 14 +++----------- internal/repository/packer_manager_test.go | 2 +- internal/repository/repository.go | 5 ++--- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/internal/repository/packer_manager.go b/internal/repository/packer_manager.go index 99d0460a0..3b2734f53 100644 --- a/internal/repository/packer_manager.go +++ b/internal/repository/packer_manager.go @@ -38,8 +38,6 @@ type packerManager struct { } const minPackSize = 4 * 1024 * 1024 -const maxPackSize = 16 * 1024 * 1024 -const maxPackers = 200 // newPackerManager returns an new packer manager which writes temporary files // to a temporary directory @@ -58,15 +56,9 @@ func (r *packerManager) findPacker(size uint) (packer *Packer, err error) { // search for a suitable packer if len(r.packers) > 0 { - debug.Log("searching packer for %d bytes\n", size) - for i, p := range r.packers { - if p.Packer.Size()+size < maxPackSize { - debug.Log("found packer %v", p) - // remove from list - r.packers = append(r.packers[:i], r.packers[i+1:]...) - return p, nil - } - } + p := r.packers[0] + r.packers = r.packers[1:] + return p, nil } // no suitable packer found, return new diff --git a/internal/repository/packer_manager_test.go b/internal/repository/packer_manager_test.go index b16e21c95..efdd7ff43 100644 --- a/internal/repository/packer_manager_test.go +++ b/internal/repository/packer_manager_test.go @@ -91,7 +91,7 @@ func fillPacks(t testing.TB, rnd *randReader, be Saver, pm *packerManager, buf [ } bytes += l - if packer.Size() < minPackSize && pm.countPacker() < maxPackers { + if packer.Size() < minPackSize { pm.insertPacker(packer) continue } diff --git a/internal/repository/repository.go b/internal/repository/repository.go index d09ddd181..7623cde83 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -191,9 +191,8 @@ func (r *Repository) SaveAndEncrypt(ctx context.Context, t restic.BlobType, data return restic.ID{}, err } - // if the pack is not full enough and there are less than maxPackers - // packers, put back to the list - if packer.Size() < minPackSize && r.countPacker() < maxPackers { + // if the pack is not full enough, put back to the list + if packer.Size() < minPackSize { debug.Log("pack is not full enough (%d bytes)", packer.Size()) r.insertPacker(packer) return *id, nil From 3541d06d076baa7ccc04a8e8576971e1429a0e73 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 16 Jul 2017 20:16:02 +0200 Subject: [PATCH 2/2] repo: Split packers for tree and data The code now bundles tree blobs and data blobs into different pack files, so we'll end up with pack files that either only contain data or trees. This is in preparation to adding a cache (#1040), because tree-only pack files can easily be cached later on. --- internal/repository/packer_manager.go | 4 +- internal/repository/packer_manager_test.go | 2 +- internal/repository/repository.go | 53 +++++++++++++++------- 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/internal/repository/packer_manager.go b/internal/repository/packer_manager.go index 3b2734f53..aa1885cc1 100644 --- a/internal/repository/packer_manager.go +++ b/internal/repository/packer_manager.go @@ -50,7 +50,7 @@ func newPackerManager(be Saver, key *crypto.Key) *packerManager { // findPacker returns a packer for a new blob of size bytes. Either a new one is // created or one is returned that already has some blobs. -func (r *packerManager) findPacker(size uint) (packer *Packer, err error) { +func (r *packerManager) findPacker() (packer *Packer, err error) { r.pm.Lock() defer r.pm.Unlock() @@ -62,7 +62,7 @@ func (r *packerManager) findPacker(size uint) (packer *Packer, err error) { } // no suitable packer found, return new - debug.Log("create new pack for %d bytes", size) + debug.Log("create new pack") tmpfile, err := fs.TempFile("", "restic-temp-pack-") if err != nil { return nil, errors.Wrap(err, "fs.TempFile") diff --git a/internal/repository/packer_manager_test.go b/internal/repository/packer_manager_test.go index efdd7ff43..aface097b 100644 --- a/internal/repository/packer_manager_test.go +++ b/internal/repository/packer_manager_test.go @@ -72,7 +72,7 @@ func fillPacks(t testing.TB, rnd *randReader, be Saver, pm *packerManager, buf [ l := rnd.rand.Intn(1 << 20) seed := rnd.rand.Int63() - packer, err := pm.findPacker(uint(l)) + packer, err := pm.findPacker() if err != nil { t.Fatal(err) } diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 7623cde83..ae35f78c0 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -24,15 +24,17 @@ type Repository struct { keyName string idx *MasterIndex - *packerManager + treePM *packerManager + dataPM *packerManager } // New returns a new repository with backend be. func New(be restic.Backend) *Repository { repo := &Repository{ - be: be, - idx: NewMasterIndex(), - packerManager: newPackerManager(be, nil), + be: be, + idx: NewMasterIndex(), + dataPM: newPackerManager(be, nil), + treePM: newPackerManager(be, nil), } return repo @@ -180,7 +182,18 @@ func (r *Repository) SaveAndEncrypt(ctx context.Context, t restic.BlobType, data } // find suitable packer and add blob - packer, err := r.findPacker(uint(len(ciphertext))) + var pm *packerManager + + switch t { + case restic.TreeBlob: + pm = r.treePM + case restic.DataBlob: + pm = r.dataPM + default: + panic(fmt.Sprintf("invalid type: %v", t)) + } + + packer, err := pm.findPacker() if err != nil { return restic.ID{}, err } @@ -194,7 +207,7 @@ func (r *Repository) SaveAndEncrypt(ctx context.Context, t restic.BlobType, data // if the pack is not full enough, put back to the list if packer.Size() < minPackSize { debug.Log("pack is not full enough (%d bytes)", packer.Size()) - r.insertPacker(packer) + pm.insertPacker(packer) return *id, nil } @@ -238,18 +251,22 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by // Flush saves all remaining packs. func (r *Repository) Flush() error { - r.pm.Lock() - defer r.pm.Unlock() + for _, pm := range []*packerManager{r.dataPM, r.treePM} { + pm.pm.Lock() - debug.Log("manually flushing %d packs", len(r.packerManager.packers)) - - for _, p := range r.packerManager.packers { - err := r.savePacker(p) - if err != nil { - return err + debug.Log("manually flushing %d packs", len(pm.packers)) + for _, p := range pm.packers { + err := r.savePacker(p) + if err != nil { + pm.pm.Unlock() + return err + } } + pm.packers = pm.packers[:0] + + pm.pm.Unlock() } - r.packerManager.packers = r.packerManager.packers[:0] + return nil } @@ -371,7 +388,8 @@ func (r *Repository) SearchKey(ctx context.Context, password string, maxKeys int } r.key = key.master - r.packerManager.key = key.master + r.dataPM.key = key.master + r.treePM.key = key.master r.keyName = key.Name() r.cfg, err = restic.LoadConfig(ctx, r) return err @@ -405,7 +423,8 @@ func (r *Repository) init(ctx context.Context, password string, cfg restic.Confi } r.key = key.master - r.packerManager.key = key.master + r.dataPM.key = key.master + r.treePM.key = key.master r.keyName = key.Name() r.cfg = cfg _, err = r.SaveJSONUnpacked(ctx, restic.ConfigFile, cfg)