From a2d42093220b6f7e3c33af40138c001dd4e7b398 Mon Sep 17 00:00:00 2001 From: greatroar <@> Date: Fri, 6 Nov 2020 20:04:52 +0100 Subject: [PATCH 1/2] Don't recurse in local backend's List if not required Due to the return if !isFile, the IsDir branch in List was never taken and subdirectories were traversed recursively. Also replaced isFile by an IsRegular check, which has been equivalent since Go 1.12 (golang/go@a2a3dd00c934fa15ad880ee5fe1f64308cbc73a7). --- internal/backend/local/local.go | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index 5261a0852..1dd0abe90 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -220,32 +220,27 @@ func (b *Local) Remove(ctx context.Context, h restic.Handle) error { return fs.Remove(fn) } -func isFile(fi os.FileInfo) bool { - return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0 -} - // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { debug.Log("List %v", t) basedir, subdirs := b.Basedir(t) + atBasedir := true err := fs.Walk(basedir, func(path string, fi os.FileInfo, err error) error { debug.Log("walk on %v\n", path) if err != nil { return err } - if path == basedir { + switch { + case atBasedir: // Skip basedir itself. + atBasedir = false return nil - } - - if !isFile(fi) { - return nil - } - - if fi.IsDir() && !subdirs { + case fi.IsDir() && !subdirs: return filepath.SkipDir + case !fi.Mode().IsRegular(): + return nil } debug.Log("send %v\n", filepath.Base(path)) From 8e213e82fc86a50faf360f74789fa6f6e79c8a17 Mon Sep 17 00:00:00 2001 From: greatroar <@> Date: Thu, 19 Nov 2020 16:46:42 +0100 Subject: [PATCH 2/2] backend/local: replace fs.Walk with custom walker This code is more strict in what it expects to find in the backend: depending on the layout, either a directory full of files or a directory full of such directories. --- internal/backend/local/local.go | 97 +++++++++++++++++++++------------ 1 file changed, 61 insertions(+), 36 deletions(-) diff --git a/internal/backend/local/local.go b/internal/backend/local/local.go index 1dd0abe90..2a48230fc 100644 --- a/internal/backend/local/local.go +++ b/internal/backend/local/local.go @@ -222,45 +222,15 @@ func (b *Local) Remove(ctx context.Context, h restic.Handle) error { // List runs fn for each file in the backend which has the type t. When an // error occurs (or fn returns an error), List stops and returns it. -func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error { +func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) (err error) { debug.Log("List %v", t) basedir, subdirs := b.Basedir(t) - atBasedir := true - err := fs.Walk(basedir, func(path string, fi os.FileInfo, err error) error { - debug.Log("walk on %v\n", path) - if err != nil { - return err - } - - switch { - case atBasedir: // Skip basedir itself. - atBasedir = false - return nil - case fi.IsDir() && !subdirs: - return filepath.SkipDir - case !fi.Mode().IsRegular(): - return nil - } - - debug.Log("send %v\n", filepath.Base(path)) - - rfi := restic.FileInfo{ - Name: filepath.Base(path), - Size: fi.Size(), - } - - if ctx.Err() != nil { - return ctx.Err() - } - - err = fn(rfi) - if err != nil { - return err - } - - return ctx.Err() - }) + if subdirs { + err = visitDirs(ctx, basedir, fn) + } else { + err = visitFiles(ctx, basedir, fn) + } if b.IsNotExist(err) { debug.Log("ignoring non-existing directory") @@ -270,6 +240,61 @@ func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.File return err } +// The following two functions are like filepath.Walk, but visit only one or +// two levels of directory structure (including dir itself as the first level). +// Also, visitDirs assumes it sees a directory full of directories, while +// visitFiles wants a directory full or regular files. +func visitDirs(ctx context.Context, dir string, fn func(restic.FileInfo) error) error { + d, err := fs.Open(dir) + if err != nil { + return err + } + defer d.Close() + + sub, err := d.Readdirnames(-1) + if err != nil { + return err + } + + for _, f := range sub { + err = visitFiles(ctx, filepath.Join(dir, f), fn) + if err != nil { + return err + } + } + return ctx.Err() +} + +func visitFiles(ctx context.Context, dir string, fn func(restic.FileInfo) error) error { + d, err := fs.Open(dir) + if err != nil { + return err + } + defer d.Close() + + sub, err := d.Readdir(-1) + if err != nil { + return err + } + + for _, fi := range sub { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + err := fn(restic.FileInfo{ + Name: fi.Name(), + Size: fi.Size(), + }) + if err != nil { + return err + } + } + return nil +} + // Delete removes the repository and all files. func (b *Local) Delete(ctx context.Context) error { debug.Log("Delete()")