From 4ae16d7661578abe284e15da82bb979deba825a7 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 24 Feb 2016 22:41:32 +0100 Subject: [PATCH 1/2] repository: Use backend.ID to load index This commit uses ParallelWorkFuncParseID() to load all indexes and ignores file names with invalid format. This fixes #475. --- src/cmds/restic/cmd_dump.go | 2 +- src/cmds/restic/cmd_rebuild_index.go | 2 +- src/restic/checker/checker.go | 4 ++-- src/restic/repository/index.go | 13 ++++--------- src/restic/repository/parallel.go | 7 +++++-- src/restic/repository/repository.go | 9 +++++---- src/restic/repository/repository_test.go | 2 +- 7 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/cmds/restic/cmd_dump.go b/src/cmds/restic/cmd_dump.go index b75d85a6e..f90fe5810 100644 --- a/src/cmds/restic/cmd_dump.go +++ b/src/cmds/restic/cmd_dump.go @@ -105,7 +105,7 @@ func (cmd CmdDump) DumpIndexes() error { for id := range cmd.repo.List(backend.Index, done) { fmt.Printf("index_id: %v\n", id) - idx, err := repository.LoadIndex(cmd.repo, id.String()) + idx, err := repository.LoadIndex(cmd.repo, id) if err != nil { return err } diff --git a/src/cmds/restic/cmd_rebuild_index.go b/src/cmds/restic/cmd_rebuild_index.go index e9d803e51..cab8bbc46 100644 --- a/src/cmds/restic/cmd_rebuild_index.go +++ b/src/cmds/restic/cmd_rebuild_index.go @@ -71,7 +71,7 @@ func (cmd CmdRebuildIndex) RebuildIndex() error { cmd.global.Printf(" loading index %v\n", i) debug.Log("RebuildIndex.RebuildIndex", "load index %v", indexID.Str()) - idx, err := repository.LoadIndex(cmd.repo, indexID.String()) + idx, err := repository.LoadIndex(cmd.repo, indexID) if err != nil { return err } diff --git a/src/restic/checker/checker.go b/src/restic/checker/checker.go index 68a8147dd..f545942d3 100644 --- a/src/restic/checker/checker.go +++ b/src/restic/checker/checker.go @@ -83,12 +83,12 @@ func (c *Checker) LoadIndex() (hints []error, errs []error) { worker := func(id backend.ID, done <-chan struct{}) error { debug.Log("LoadIndex", "worker got index %v", id) - idx, err := repository.LoadIndexWithDecoder(c.repo, id.String(), repository.DecodeIndex) + idx, err := repository.LoadIndexWithDecoder(c.repo, id, repository.DecodeIndex) if err == repository.ErrOldIndexFormat { debug.Log("LoadIndex", "index %v has old format", id.Str()) hints = append(hints, ErrOldIndexFormat{id}) - idx, err = repository.LoadIndexWithDecoder(c.repo, id.String(), repository.DecodeOldIndex) + idx, err = repository.LoadIndexWithDecoder(c.repo, id, repository.DecodeOldIndex) } if err != nil { diff --git a/src/restic/repository/index.go b/src/restic/repository/index.go index c5c75a403..56534bd9d 100644 --- a/src/restic/repository/index.go +++ b/src/restic/repository/index.go @@ -557,15 +557,10 @@ func DecodeOldIndex(rd io.Reader) (idx *Index, err error) { } // LoadIndexWithDecoder loads the index and decodes it with fn. -func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Index, error)) (idx *Index, err error) { +func LoadIndexWithDecoder(repo *Repository, id backend.ID, fn func(io.Reader) (*Index, error)) (idx *Index, err error) { debug.Log("LoadIndexWithDecoder", "Loading index %v", id[:8]) - idxID, err := backend.ParseID(id) - if err != nil { - return nil, err - } - - buf, err := repo.LoadAndDecrypt(backend.Index, idxID) + buf, err := repo.LoadAndDecrypt(backend.Index, id) if err != nil { return nil, err } @@ -576,7 +571,7 @@ func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Inde return nil, err } - idx.id = idxID + idx.id = id return idx, nil } @@ -588,7 +583,7 @@ func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Inde func ConvertIndex(repo *Repository, id backend.ID) (backend.ID, error) { debug.Log("ConvertIndex", "checking index %v", id.Str()) - idx, err := LoadIndexWithDecoder(repo, id.String(), DecodeOldIndex) + idx, err := LoadIndexWithDecoder(repo, id, DecodeOldIndex) if err != nil { debug.Log("ConvertIndex", "LoadIndexWithDecoder(%v) returned error: %v", id.Str(), err) return id, err diff --git a/src/restic/repository/parallel.go b/src/restic/repository/parallel.go index ff659f1fe..d1e66c58a 100644 --- a/src/restic/repository/parallel.go +++ b/src/restic/repository/parallel.go @@ -4,6 +4,7 @@ import ( "sync" "restic/backend" + "restic/debug" ) func closeIfOpen(ch chan struct{}) { @@ -75,12 +76,14 @@ func FilesInParallel(repo backend.Lister, t backend.Type, n uint, f ParallelWork } // ParallelWorkFuncParseID converts a function that takes a backend.ID to a -// function that takes a string. +// function that takes a string. Filenames that do not parse as a backend.ID +// are ignored. func ParallelWorkFuncParseID(f ParallelIDWorkFunc) ParallelWorkFunc { return func(s string, done <-chan struct{}) error { id, err := backend.ParseID(s) if err != nil { - return err + debug.Log("repository.ParallelWorkFuncParseID", "invalid ID %q: %v", id, err) + return nil } return f(id, done) diff --git a/src/restic/repository/repository.go b/src/restic/repository/repository.go index cc34682e8..f981f5b76 100644 --- a/src/restic/repository/repository.go +++ b/src/restic/repository/repository.go @@ -366,7 +366,7 @@ func (r *Repository) LoadIndex() error { errCh := make(chan error, 1) indexes := make(chan *Index) - worker := func(id string, done <-chan struct{}) error { + worker := func(id backend.ID, done <-chan struct{}) error { idx, err := LoadIndex(r, id) if err != nil { return err @@ -382,7 +382,8 @@ func (r *Repository) LoadIndex() error { go func() { defer close(indexes) - errCh <- FilesInParallel(r.be, backend.Index, loadIndexParallelism, worker) + errCh <- FilesInParallel(r.be, backend.Index, loadIndexParallelism, + ParallelWorkFuncParseID(worker)) }() for idx := range indexes { @@ -397,14 +398,14 @@ func (r *Repository) LoadIndex() error { } // LoadIndex loads the index id from backend and returns it. -func LoadIndex(repo *Repository, id string) (*Index, error) { +func LoadIndex(repo *Repository, id backend.ID) (*Index, error) { idx, err := LoadIndexWithDecoder(repo, id, DecodeIndex) if err == nil { return idx, nil } if err == ErrOldIndexFormat { - fmt.Fprintf(os.Stderr, "index %v has old format\n", id[:10]) + fmt.Fprintf(os.Stderr, "index %v has old format\n", id.Str()) return LoadIndexWithDecoder(repo, id, DecodeOldIndex) } diff --git a/src/restic/repository/repository_test.go b/src/restic/repository/repository_test.go index 3ca9582df..5bd595086 100644 --- a/src/restic/repository/repository_test.go +++ b/src/restic/repository/repository_test.go @@ -292,7 +292,7 @@ func TestRepositoryIncrementalIndex(t *testing.T) { packEntries := make(map[backend.ID]map[backend.ID]struct{}) for id := range repo.List(backend.Index, nil) { - idx, err := repository.LoadIndex(repo, id.String()) + idx, err := repository.LoadIndex(repo, id) OK(t, err) for pb := range idx.Each(nil) { From 17f5b524a6fa544314a6b8e64257a8fa004e0701 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 24 Feb 2016 22:43:04 +0100 Subject: [PATCH 2/2] local: Replace matching code with proper Readdir() --- src/restic/backend/local/local.go | 73 ++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 10 deletions(-) diff --git a/src/restic/backend/local/local.go b/src/restic/backend/local/local.go index 8bf4f1803..c0f33d036 100644 --- a/src/restic/backend/local/local.go +++ b/src/restic/backend/local/local.go @@ -235,31 +235,84 @@ func (b *Local) Remove(t backend.Type, name string) error { return os.Remove(fn) } +func isFile(fi os.FileInfo) bool { + return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0 +} + +func readdir(d string) (fileInfos []os.FileInfo, err error) { + f, e := os.Open(d) + if e != nil { + return nil, e + } + + defer func() { + e := f.Close() + if err == nil { + err = e + } + }() + + return f.Readdir(-1) +} + +// listDir returns a list of all files in d. +func listDir(d string) (filenames []string, err error) { + fileInfos, err := readdir(d) + if err != nil { + return nil, err + } + + for _, fi := range fileInfos { + if isFile(fi) { + filenames = append(filenames, fi.Name()) + } + } + + return filenames, nil +} + +// listDirs returns a list of all files in directories within d. +func listDirs(dir string) (filenames []string, err error) { + fileInfos, err := readdir(dir) + if err != nil { + return nil, err + } + + for _, fi := range fileInfos { + if !fi.IsDir() { + continue + } + + files, err := listDir(filepath.Join(dir, fi.Name())) + if err != nil { + continue + } + + filenames = append(filenames, files...) + } + + return filenames, nil +} + // List returns a channel that yields all names of blobs of type t. A // goroutine is started for this. If the channel done is closed, sending // stops. func (b *Local) List(t backend.Type, done <-chan struct{}) <-chan string { - var pattern string + lister := listDir if t == backend.Data { - pattern = filepath.Join(dirname(b.p, t, ""), "*", "*") - } else { - pattern = filepath.Join(dirname(b.p, t, ""), "*") + lister = listDirs } ch := make(chan string) - matches, err := filepath.Glob(pattern) + items, err := lister(filepath.Join(dirname(b.p, t, ""))) if err != nil { close(ch) return ch } - for i := range matches { - matches[i] = filepath.Base(matches[i]) - } - go func() { defer close(ch) - for _, m := range matches { + for _, m := range items { if m == "" { continue }