From d36f07c6eb2bc43808a49d16942cc43e2ea70d87 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 12 Jul 2015 17:09:48 +0200 Subject: [PATCH] checker: add option to remove orphaned packs --- checker/checker.go | 22 ++++++++++++++------- cmd/restic/cmd_check.go | 44 +++++++++++++++++++++++++++++++---------- 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/checker/checker.go b/checker/checker.go index 0238a53b3..852c85a43 100644 --- a/checker/checker.go +++ b/checker/checker.go @@ -48,7 +48,8 @@ type Checker struct { sync.Mutex M map[mapID]uint } - indexes map[mapID]*repository.Index + indexes map[mapID]*repository.Index + orphanedPacks backend.IDs masterIndex *repository.Index @@ -139,12 +140,13 @@ func (c *Checker) LoadIndex() error { // PackError describes an error with a specific pack. type PackError struct { - ID backend.ID - error + ID backend.ID + Orphaned bool + Err error } func (e PackError) Error() string { - return "pack " + e.ID.String() + ": " + e.error.Error() + return "pack " + e.ID.String() + ": " + e.Err.Error() } func packIDTester(repo *repository.Repository, inChan <-chan mapID, errChan chan<- error, wg *sync.WaitGroup, done <-chan struct{}) { @@ -156,10 +158,10 @@ func packIDTester(repo *repository.Repository, inChan <-chan mapID, errChan chan for id := range inChan { ok, err := repo.Backend().Test(backend.Data, map2str(id)) if err != nil { - err = PackError{map2id(id), err} + err = PackError{ID: map2id(id), Err: err} } else { if !ok { - err = PackError{map2id(id), errors.New("does not exist")} + err = PackError{ID: map2id(id), Err: errors.New("does not exist")} } } @@ -227,10 +229,11 @@ func (c *Checker) Packs(errChan chan<- error, done <-chan struct{}) { for id := range c.repo.List(backend.Data, done) { debug.Log("Checker.Packs", "check data blob %v", id.Str()) if _, ok := seenPacks[id2map(id)]; !ok { + c.orphanedPacks = append(c.orphanedPacks, id) select { case <-done: return - case errChan <- PackError{id, errors.New("not referenced in any index")}: + case errChan <- PackError{ID: id, Orphaned: true, Err: errors.New("not referenced in any index")}: } } } @@ -591,3 +594,8 @@ func (c *Checker) UnusedBlobs() (blobs backend.IDs) { return blobs } + +// OrphanedPacks returns a slice of unused packs (only available after Packs() was run). +func (c *Checker) OrphanedPacks() backend.IDs { + return c.orphanedPacks +} diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index ccf65d4d7..d47de1c7b 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -5,11 +5,13 @@ import ( "fmt" "os" + "github.com/restic/restic/backend" "github.com/restic/restic/checker" ) type CmdCheck struct { - ReadData bool ` long:"read-data" description:"Read data blobs" default:"false"` + ReadData bool `long:"read-data" description:"Read data blobs" default:"false"` + RemoveOrphaned bool `long:"remove" description:"Remove data that isn't used" default:"false"` global *GlobalOptions } @@ -45,10 +47,10 @@ func (cmd CmdCheck) Execute(args []string) error { return err } - checker := checker.New(repo) + chkr := checker.New(repo) cmd.global.Verbosef("Load indexes\n") - if err = checker.LoadIndex(); err != nil { + if err = chkr.LoadIndex(); err != nil { return err } @@ -59,20 +61,42 @@ func (cmd CmdCheck) Execute(args []string) error { errChan := make(chan error) cmd.global.Verbosef("Check all packs\n") - go checker.Packs(errChan, done) + go chkr.Packs(errChan, done) + + foundOrphanedPacks := false + for err := range errChan { + errorsFound = true + fmt.Fprintf(os.Stderr, "%v\n", err) + + if e, ok := err.(checker.PackError); ok && e.Orphaned { + foundOrphanedPacks = true + } + } + + cmd.global.Verbosef("Check snapshots, trees and blobs\n") + errChan = make(chan error) + go chkr.Structure(errChan, done) for err := range errChan { errorsFound = true fmt.Fprintf(os.Stderr, "error: %v\n", err) } - cmd.global.Verbosef("Check snapshots, trees and blobs\n") - errChan = make(chan error) - go checker.Structure(errChan, done) + for _, id := range chkr.UnusedBlobs() { + cmd.global.Verbosef("unused blob %v\n", id.Str()) + } - for err := range errChan { - errorsFound = true - fmt.Fprintf(os.Stderr, "error: %v\n", err) + if foundOrphanedPacks && cmd.RemoveOrphaned { + IDs := chkr.OrphanedPacks() + cmd.global.Verbosef("Remove %d orphaned packs... ", len(IDs)) + + for _, id := range IDs { + if err := repo.Backend().Remove(backend.Data, id.String()); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + } + } + + cmd.global.Verbosef("done\n") } if errorsFound {