From f0f17db8474cd5c97793c19e663b56fa4f5c1356 Mon Sep 17 00:00:00 2001 From: Fabian Wickborn Date: Wed, 4 Oct 2017 20:30:43 +0200 Subject: [PATCH] Cache evaluated directories in isExcludedByFile Fixes #1271. --- cmd/restic/cmd_backup.go | 3 +- cmd/restic/exclude.go | 72 +++++++++++++++++++++++++++++++++++--- cmd/restic/exclude_test.go | 2 +- 3 files changed, 70 insertions(+), 7 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 0c254738a..26e5434f1 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -367,8 +367,9 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error { opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55") } + rc := &rejectionCache{} for _, spec := range opts.ExcludeIfPresent { - f, err := rejectIfPresent(spec) + f, err := rejectIfPresent(spec, rc) if err != nil { return err } diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go index bb14bfafb..0a6d8bcec 100644 --- a/cmd/restic/exclude.go +++ b/cmd/restic/exclude.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "strings" + "sync" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" @@ -15,6 +16,50 @@ import ( "github.com/restic/restic/internal/repository" ) +type rejectionCache struct { + m map[string]bool + mtx sync.Mutex +} + +// Lock locks the mutex in rc. +func (rc *rejectionCache) Lock() { + if rc != nil { + rc.mtx.Lock() + } +} + +// Unlock unlocks the mutex in rc. +func (rc *rejectionCache) Unlock() { + if rc != nil { + rc.mtx.Unlock() + } +} + +// Get returns the last stored value for dir and a second boolean that +// indicates whether that value was actually written to the cache. It is the +// callers responsibility to call rc.Lock and rc.Unlock before using this +// method, otherwise data races may occur. +func (rc *rejectionCache) Get(dir string) (bool, bool) { + if rc == nil || rc.m == nil { + return false, false + } + v, ok := rc.m[dir] + return v, ok +} + +// Store stores a new value for dir. It is the callers responsibility to call +// rc.Lock and rc.Unlock before using this method, otherwise data races may +// occur. +func (rc *rejectionCache) Store(dir string, rejected bool) { + if rc == nil { + return + } + if rc.m == nil { + rc.m = make(map[string]bool) + } + rc.m[dir] = rejected +} + // RejectFunc is a function that takes a filename and os.FileInfo of a // file that would be included in the backup. The function returns true if it // should be excluded (rejected) from the backup. @@ -42,8 +87,10 @@ func rejectByPattern(patterns []string) RejectFunc { // should be excluded. The RejectFunc considers a file to be excluded when // it resides in a directory with an exclusion file, that is specified by // excludeFileSpec in the form "filename[:content]". The returned error is -// non-nil if the filename component of excludeFileSpec is empty. -func rejectIfPresent(excludeFileSpec string) (RejectFunc, error) { +// non-nil if the filename component of excludeFileSpec is empty. If rc is +// non-nil, it is going to be used in the RejectFunc to expedite the evaluation +// of a directory based on previous visits. +func rejectIfPresent(excludeFileSpec string, rc *rejectionCache) (RejectFunc, error) { if excludeFileSpec == "" { return nil, errors.New("name for exclusion tagfile is empty") } @@ -60,15 +107,17 @@ func rejectIfPresent(excludeFileSpec string) (RejectFunc, error) { } debug.Log("using %q as exclusion tagfile", tf) fn := func(filename string, _ os.FileInfo) bool { - return isExcludedByFile(filename, tf, tc) + return isExcludedByFile(filename, tf, tc, rc) } return fn, nil } // isExcludedByFile interprets filename as a path and returns true if that file // is in a excluded directory. A directory is identified as excluded if it contains a -// tagfile which bears the name specified in tagFilename and starts with header. -func isExcludedByFile(filename, tagFilename, header string) bool { +// tagfile which bears the name specified in tagFilename and starts with +// header. If rc is non-nil, it is used to expedite the evaluation of a +// directory based on previous visits. +func isExcludedByFile(filename, tagFilename, header string, rc *rejectionCache) bool { if tagFilename == "" { return false } @@ -76,6 +125,19 @@ func isExcludedByFile(filename, tagFilename, header string) bool { if base == tagFilename { return false // do not exclude the tagfile itself } + rc.Lock() + defer rc.Unlock() + + rejected, visited := rc.Get(dir) + if visited { + return rejected + } + rejected = isDirExcludedByFile(dir, tagFilename, header) + rc.Store(dir, rejected) + return rejected +} + +func isDirExcludedByFile(dir, tagFilename, header string) bool { tf := filepath.Join(dir, tagFilename) _, err := fs.Lstat(tf) if os.IsNotExist(err) { diff --git a/cmd/restic/exclude_test.go b/cmd/restic/exclude_test.go index ec9b0bade..5cadf6e9b 100644 --- a/cmd/restic/exclude_test.go +++ b/cmd/restic/exclude_test.go @@ -76,7 +76,7 @@ func TestIsExcludedByFile(t *testing.T) { if tc.content == "" { h = "" } - if got := isExcludedByFile(foo, tagFilename, h); tc.want != got { + if got := isExcludedByFile(foo, tagFilename, h, nil); tc.want != got { t.Fatalf("expected %v, got %v", tc.want, got) } })