From ea75509d6ecd4d15b723bbbf6fb9f39292f7e8ec Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 9 Sep 2017 21:12:41 +0200 Subject: [PATCH 01/14] Print warning for non-existing items --- cmd/restic/cmd_backup.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 09a4ba2f5..86cd589f7 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -216,6 +216,7 @@ func filterExisting(items []string) (result []string, err error) { for _, item := range items { _, err := fs.Lstat(item) if err != nil && os.IsNotExist(errors.Cause(err)) { + Warnf("%v does not exist, skipping\n", item) continue } From c22c582546976a5153cf41b95231fdb7ec3d0d4c Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 9 Sep 2017 21:24:29 +0200 Subject: [PATCH 02/14] Allow multiple exclude-if-present --- cmd/restic/cmd_backup.go | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 86cd589f7..501aaf03e 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -59,7 +59,7 @@ type BackupOptions struct { Excludes []string ExcludeFiles []string ExcludeOtherFS bool - ExcludeIfPresent string + ExcludeIfPresent []string ExcludeCaches bool Stdin bool StdinFilename string @@ -79,7 +79,7 @@ func init() { f.StringArrayVarP(&backupOptions.Excludes, "exclude", "e", nil, "exclude a `pattern` (can be specified multiple times)") f.StringArrayVar(&backupOptions.ExcludeFiles, "exclude-file", nil, "read exclude patterns from a `file` (can be specified multiple times)") f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "exclude other file systems") - f.StringVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", "", "takes filename[:header], exclude contents of directories containing filename (except filename itself) if header of that file is as provided") + f.StringArrayVar(&backupOptions.ExcludeIfPresent, "exclude-if-present", nil, "takes filename[:header], exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times)") f.BoolVar(&backupOptions.ExcludeCaches, "exclude-caches", false, `excludes cache directories that are marked with a CACHEDIR.TAG file`) f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin") f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "file name to use when reading from stdin") @@ -423,15 +423,17 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error { } if opts.ExcludeCaches { - if opts.ExcludeIfPresent != "" { - return fmt.Errorf("cannot have --exclude-caches defined at the same time as --exclude-if-present") - } - opts.ExcludeIfPresent = "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55" + opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55") } - excludeByFile, err := excludeByFile(opts.ExcludeIfPresent) - if err != nil { - return err + var excludesByFile []FilenameCheck + for _, spec := range opts.ExcludeIfPresent { + f, err := excludeByFile(spec) + if err != nil { + return err + } + + excludesByFile = append(excludesByFile, f) } selectFilter := func(item string, fi os.FileInfo) bool { @@ -445,9 +447,11 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error { return false } - if excludeByFile(item) { - debug.Log("path %q excluded by tagfile", item) - return false + for _, excludeByFile := range excludesByFile { + if excludeByFile(item) { + debug.Log("path %q excluded by tagfile", item) + return false + } } if !opts.ExcludeOtherFS || fi == nil { From d937ad8cf67bebb2bbdabb60bb51d84a9773eefb Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 10 Sep 2017 14:18:25 +0200 Subject: [PATCH 03/14] Rename FilenameCheck to RejectFunc We already have the opposite: pipe.SelectFunc(item string, fi os.FileInfo) bool, so RejectFunc is a good name. --- cmd/restic/cmd_backup.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 501aaf03e..a1a68f7fa 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -426,7 +426,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error { opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55") } - var excludesByFile []FilenameCheck + var excludesByFile []RejectFunc for _, spec := range opts.ExcludeIfPresent { f, err := excludeByFile(spec) if err != nil { @@ -448,7 +448,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error { } for _, excludeByFile := range excludesByFile { - if excludeByFile(item) { + if excludeByFile(item, fi) { debug.Log("path %q excluded by tagfile", item) return false } @@ -550,18 +550,19 @@ func readExcludePatternsFromFiles(excludeFiles []string) []string { return excludes } -// FilenameCheck is a function that takes a filename and returns a boolean -// depending on arbitrary check. -type FilenameCheck func(filename string) bool +// RejectFunc is a function that takes a filename and os.FileInfo of a +// file that would be included in the backup. The function returns true if it +// should be excluded (rejected) from the backup. +type RejectFunc func(filename string, fi os.FileInfo) bool // excludeByFile returns a FilenameCheck which itself returns whether a path // should be excluded. The FilenameCheck considers a file to be excluded when // it resides in a directory with an exclusion file, that is specified by // excludeFileSpec in the form "filename[:content]". The returned error is // non-nil if the filename component of excludeFileSpec is empty. -func excludeByFile(excludeFileSpec string) (FilenameCheck, error) { +func excludeByFile(excludeFileSpec string) (RejectFunc, error) { if excludeFileSpec == "" { - return func(string) bool { return false }, nil + return func(string, os.FileInfo) bool { return false }, nil } colon := strings.Index(excludeFileSpec, ":") if colon == 0 { @@ -575,7 +576,7 @@ func excludeByFile(excludeFileSpec string) (FilenameCheck, error) { tf = excludeFileSpec } debug.Log("using %q as exclusion tagfile", tf) - fn := func(filename string) bool { + fn := func(filename string, _ os.FileInfo) bool { return isExcludedByFile(filename, tf, tc) } return fn, nil From a9c705009ceaefd469a9da524031b99a712f2c39 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 10 Sep 2017 14:23:29 +0200 Subject: [PATCH 04/14] Move reject functions to new file --- cmd/restic/cmd_backup.go | 85 ----------------------------------- cmd/restic/exclude.go | 97 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 85 deletions(-) create mode 100644 cmd/restic/exclude.go diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index a1a68f7fa..6da6af333 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -2,7 +2,6 @@ package main import ( "bufio" - "bytes" "context" "fmt" "io" @@ -549,87 +548,3 @@ func readExcludePatternsFromFiles(excludeFiles []string) []string { } return excludes } - -// RejectFunc is a function that takes a filename and os.FileInfo of a -// file that would be included in the backup. The function returns true if it -// should be excluded (rejected) from the backup. -type RejectFunc func(filename string, fi os.FileInfo) bool - -// excludeByFile returns a FilenameCheck which itself returns whether a path -// should be excluded. The FilenameCheck considers a file to be excluded when -// it resides in a directory with an exclusion file, that is specified by -// excludeFileSpec in the form "filename[:content]". The returned error is -// non-nil if the filename component of excludeFileSpec is empty. -func excludeByFile(excludeFileSpec string) (RejectFunc, error) { - if excludeFileSpec == "" { - return func(string, os.FileInfo) bool { return false }, nil - } - colon := strings.Index(excludeFileSpec, ":") - if colon == 0 { - return nil, fmt.Errorf("no name for exclusion tagfile provided") - } - tf, tc := "", "" - if colon > 0 { - tf = excludeFileSpec[:colon] - tc = excludeFileSpec[colon+1:] - } else { - tf = excludeFileSpec - } - debug.Log("using %q as exclusion tagfile", tf) - fn := func(filename string, _ os.FileInfo) bool { - return isExcludedByFile(filename, tf, tc) - } - return fn, nil -} - -// isExcludedByFile interprets filename as a path and returns true if that file -// is in a excluded directory. A directory is identified as excluded if it contains a -// tagfile which bears the name specified in tagFilename and starts with header. -func isExcludedByFile(filename, tagFilename, header string) bool { - if tagFilename == "" { - return false - } - dir, base := filepath.Split(filename) - if base == tagFilename { - return false // do not exclude the tagfile itself - } - tf := filepath.Join(dir, tagFilename) - _, err := fs.Lstat(tf) - if os.IsNotExist(err) { - return false - } - if err != nil { - Warnf("could not access exclusion tagfile: %v", err) - return false - } - // when no signature is given, the mere presence of tf is enough reason - // to exclude filename - if len(header) == 0 { - return true - } - // From this stage, errors mean tagFilename exists but it is malformed. - // Warnings will be generated so that the user is informed that the - // indented ignore-action is not performed. - f, err := os.Open(tf) - if err != nil { - Warnf("could not open exclusion tagfile: %v", err) - return false - } - defer f.Close() - buf := make([]byte, len(header)) - _, err = io.ReadFull(f, buf) - // EOF is handled with a dedicated message, otherwise the warning were too cryptic - if err == io.EOF { - Warnf("invalid (too short) signature in exclusion tagfile %q\n", tf) - return false - } - if err != nil { - Warnf("could not read signature from exclusion tagfile %q: %v\n", tf, err) - return false - } - if bytes.Compare(buf, []byte(header)) != 0 { - Warnf("invalid signature in exclusion tagfile %q\n", tf) - return false - } - return true -} diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go new file mode 100644 index 000000000..e8cb0daed --- /dev/null +++ b/cmd/restic/exclude.go @@ -0,0 +1,97 @@ +package main + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/fs" +) + +// RejectFunc is a function that takes a filename and os.FileInfo of a +// file that would be included in the backup. The function returns true if it +// should be excluded (rejected) from the backup. +type RejectFunc func(filename string, fi os.FileInfo) bool + +// excludeByFile returns a RejectFunc which itself returns whether a path +// should be excluded. The RejectFunc considers a file to be excluded when +// it resides in a directory with an exclusion file, that is specified by +// excludeFileSpec in the form "filename[:content]". The returned error is +// non-nil if the filename component of excludeFileSpec is empty. +func excludeByFile(excludeFileSpec string) (RejectFunc, error) { + if excludeFileSpec == "" { + return func(string, os.FileInfo) bool { return false }, nil + } + colon := strings.Index(excludeFileSpec, ":") + if colon == 0 { + return nil, fmt.Errorf("no name for exclusion tagfile provided") + } + tf, tc := "", "" + if colon > 0 { + tf = excludeFileSpec[:colon] + tc = excludeFileSpec[colon+1:] + } else { + tf = excludeFileSpec + } + debug.Log("using %q as exclusion tagfile", tf) + fn := func(filename string, _ os.FileInfo) bool { + return isExcludedByFile(filename, tf, tc) + } + return fn, nil +} + +// isExcludedByFile interprets filename as a path and returns true if that file +// is in a excluded directory. A directory is identified as excluded if it contains a +// tagfile which bears the name specified in tagFilename and starts with header. +func isExcludedByFile(filename, tagFilename, header string) bool { + if tagFilename == "" { + return false + } + dir, base := filepath.Split(filename) + if base == tagFilename { + return false // do not exclude the tagfile itself + } + tf := filepath.Join(dir, tagFilename) + _, err := fs.Lstat(tf) + if os.IsNotExist(err) { + return false + } + if err != nil { + Warnf("could not access exclusion tagfile: %v", err) + return false + } + // when no signature is given, the mere presence of tf is enough reason + // to exclude filename + if len(header) == 0 { + return true + } + // From this stage, errors mean tagFilename exists but it is malformed. + // Warnings will be generated so that the user is informed that the + // indented ignore-action is not performed. + f, err := os.Open(tf) + if err != nil { + Warnf("could not open exclusion tagfile: %v", err) + return false + } + defer f.Close() + buf := make([]byte, len(header)) + _, err = io.ReadFull(f, buf) + // EOF is handled with a dedicated message, otherwise the warning were too cryptic + if err == io.EOF { + Warnf("invalid (too short) signature in exclusion tagfile %q\n", tf) + return false + } + if err != nil { + Warnf("could not read signature from exclusion tagfile %q: %v\n", tf, err) + return false + } + if bytes.Compare(buf, []byte(header)) != 0 { + Warnf("invalid signature in exclusion tagfile %q\n", tf) + return false + } + return true +} From 4a0129fc2bef2c057aa26eadc3c399915e3e76dd Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 10 Sep 2017 14:25:58 +0200 Subject: [PATCH 05/14] Rename excludeByFile -> rejectIfPresent --- cmd/restic/cmd_backup.go | 2 +- cmd/restic/exclude.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 6da6af333..413dd8862 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -427,7 +427,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error { var excludesByFile []RejectFunc for _, spec := range opts.ExcludeIfPresent { - f, err := excludeByFile(spec) + f, err := rejectIfPresent(spec) if err != nil { return err } diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go index e8cb0daed..34d3971f1 100644 --- a/cmd/restic/exclude.go +++ b/cmd/restic/exclude.go @@ -17,12 +17,12 @@ import ( // should be excluded (rejected) from the backup. type RejectFunc func(filename string, fi os.FileInfo) bool -// excludeByFile returns a RejectFunc which itself returns whether a path +// rejectIfPresent returns a RejectFunc which itself returns whether a path // should be excluded. The RejectFunc considers a file to be excluded when // it resides in a directory with an exclusion file, that is specified by // excludeFileSpec in the form "filename[:content]". The returned error is // non-nil if the filename component of excludeFileSpec is empty. -func excludeByFile(excludeFileSpec string) (RejectFunc, error) { +func rejectIfPresent(excludeFileSpec string) (RejectFunc, error) { if excludeFileSpec == "" { return func(string, os.FileInfo) bool { return false }, nil } From 0dfdf028852215bf2ac13981fda43cbfca6e63bb Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 10 Sep 2017 14:34:28 +0200 Subject: [PATCH 06/14] Rework pattern excludes --- cmd/restic/cmd_backup.go | 26 ++++++++++---------------- cmd/restic/exclude.go | 22 +++++++++++++++++++++- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 413dd8862..3ae109085 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -15,7 +15,6 @@ import ( "github.com/restic/restic/internal/archiver" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/restic" ) @@ -416,39 +415,34 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error { Verbosef("scan %v\n", target) + // rejectFuncs collect functions that can reject items from the backup + var rejectFuncs []RejectFunc + // add patterns from file if len(opts.ExcludeFiles) > 0 { opts.Excludes = append(opts.Excludes, readExcludePatternsFromFiles(opts.ExcludeFiles)...) } + if len(opts.Excludes) > 0 { + rejectFuncs = append(rejectFuncs, rejectByPattern(opts.Excludes)) + } + if opts.ExcludeCaches { opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55") } - var excludesByFile []RejectFunc for _, spec := range opts.ExcludeIfPresent { f, err := rejectIfPresent(spec) if err != nil { return err } - excludesByFile = append(excludesByFile, f) + rejectFuncs = append(rejectFuncs, f) } selectFilter := func(item string, fi os.FileInfo) bool { - matched, _, err := filter.List(opts.Excludes, item) - if err != nil { - Warnf("error for exclude pattern: %v", err) - } - - if matched { - debug.Log("path %q excluded by a filter", item) - return false - } - - for _, excludeByFile := range excludesByFile { - if excludeByFile(item, fi) { - debug.Log("path %q excluded by tagfile", item) + for _, reject := range rejectFuncs { + if reject(item, fi) { return false } } diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go index 34d3971f1..294a8e3eb 100644 --- a/cmd/restic/exclude.go +++ b/cmd/restic/exclude.go @@ -9,6 +9,8 @@ import ( "strings" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/filter" "github.com/restic/restic/internal/fs" ) @@ -17,6 +19,24 @@ import ( // should be excluded (rejected) from the backup. type RejectFunc func(filename string, fi os.FileInfo) bool +// rejectByPattern returns a RejectFunc which rejects files that match +// one of the patterns. +func rejectByPattern(patterns []string) RejectFunc { + return func(item string, fi os.FileInfo) bool { + matched, _, err := filter.List(patterns, item) + if err != nil { + Warnf("error for exclude pattern: %v", err) + } + + if matched { + debug.Log("path %q excluded by a filter", item) + return true + } + + return false + } +} + // rejectIfPresent returns a RejectFunc which itself returns whether a path // should be excluded. The RejectFunc considers a file to be excluded when // it resides in a directory with an exclusion file, that is specified by @@ -24,7 +44,7 @@ type RejectFunc func(filename string, fi os.FileInfo) bool // non-nil if the filename component of excludeFileSpec is empty. func rejectIfPresent(excludeFileSpec string) (RejectFunc, error) { if excludeFileSpec == "" { - return func(string, os.FileInfo) bool { return false }, nil + return nil, errors.New("name for exclusion tagfile is empty") } colon := strings.Index(excludeFileSpec, ":") if colon == 0 { From e117f613af35c063df38ec6df25e0e18fc2015d6 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 10 Sep 2017 15:13:40 +0200 Subject: [PATCH 07/14] Move device test into new RejectFunc --- cmd/restic/cmd_backup.go | 106 +++++++++++---------------------------- cmd/restic/exclude.go | 62 +++++++++++++++++++++++ 2 files changed, 90 insertions(+), 78 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 3ae109085..a994717f4 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -228,27 +228,6 @@ func filterExisting(items []string) (result []string, err error) { return } -// gatherDevices returns the set of unique device ids of the files and/or -// directory paths listed in "items". -func gatherDevices(items []string) (deviceMap map[string]uint64, err error) { - deviceMap = make(map[string]uint64) - for _, item := range items { - fi, err := fs.Lstat(item) - if err != nil { - return nil, err - } - id, err := fs.DeviceID(fi) - if err != nil { - return nil, err - } - deviceMap[item] = id - } - if len(deviceMap) == 0 { - return nil, errors.New("zero allowed devices") - } - return deviceMap, nil -} - func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string) error { if len(args) != 0 { return errors.Fatal("when reading from stdin, no additional files can be specified") @@ -361,14 +340,38 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error { return err } + // rejectFuncs collect functions that can reject items from the backup + var rejectFuncs []RejectFunc + // allowed devices - var allowedDevs map[string]uint64 if opts.ExcludeOtherFS { - allowedDevs, err = gatherDevices(target) + f, err := rejectByDevice(target) if err != nil { return err } - debug.Log("allowed devices: %v\n", allowedDevs) + rejectFuncs = append(rejectFuncs, f) + } + + // add patterns from file + if len(opts.ExcludeFiles) > 0 { + opts.Excludes = append(opts.Excludes, readExcludePatternsFromFiles(opts.ExcludeFiles)...) + } + + if len(opts.Excludes) > 0 { + rejectFuncs = append(rejectFuncs, rejectByPattern(opts.Excludes)) + } + + if opts.ExcludeCaches { + opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55") + } + + for _, spec := range opts.ExcludeIfPresent { + f, err := rejectIfPresent(spec) + if err != nil { + return err + } + + rejectFuncs = append(rejectFuncs, f) } repo, err := OpenRepository(gopts) @@ -415,66 +418,13 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error { Verbosef("scan %v\n", target) - // rejectFuncs collect functions that can reject items from the backup - var rejectFuncs []RejectFunc - - // add patterns from file - if len(opts.ExcludeFiles) > 0 { - opts.Excludes = append(opts.Excludes, readExcludePatternsFromFiles(opts.ExcludeFiles)...) - } - - if len(opts.Excludes) > 0 { - rejectFuncs = append(rejectFuncs, rejectByPattern(opts.Excludes)) - } - - if opts.ExcludeCaches { - opts.ExcludeIfPresent = append(opts.ExcludeIfPresent, "CACHEDIR.TAG:Signature: 8a477f597d28d172789f06886806bc55") - } - - for _, spec := range opts.ExcludeIfPresent { - f, err := rejectIfPresent(spec) - if err != nil { - return err - } - - rejectFuncs = append(rejectFuncs, f) - } - selectFilter := func(item string, fi os.FileInfo) bool { for _, reject := range rejectFuncs { if reject(item, fi) { return false } } - - if !opts.ExcludeOtherFS || fi == nil { - return true - } - - id, err := fs.DeviceID(fi) - if err != nil { - // This should never happen because gatherDevices() would have - // errored out earlier. If it still does that's a reason to panic. - panic(err) - } - - for dir := item; dir != ""; dir = filepath.Dir(dir) { - debug.Log("item %v, test dir %v", item, dir) - - allowedID, ok := allowedDevs[dir] - if !ok { - continue - } - - if allowedID != id { - debug.Log("path %q on disallowed device %d", item, id) - return false - } - - return true - } - - panic(fmt.Sprintf("item %v, device id %v not found, allowedDevs: %v", item, id, allowedDevs)) + return true } stat, err := archiver.Scan(target, selectFilter, newScanProgress(gopts)) diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go index 294a8e3eb..d520f9c80 100644 --- a/cmd/restic/exclude.go +++ b/cmd/restic/exclude.go @@ -115,3 +115,65 @@ func isExcludedByFile(filename, tagFilename, header string) bool { } return true } + +// gatherDevices returns the set of unique device ids of the files and/or +// directory paths listed in "items". +func gatherDevices(items []string) (deviceMap map[string]uint64, err error) { + deviceMap = make(map[string]uint64) + for _, item := range items { + fi, err := fs.Lstat(item) + if err != nil { + return nil, err + } + id, err := fs.DeviceID(fi) + if err != nil { + return nil, err + } + deviceMap[item] = id + } + if len(deviceMap) == 0 { + return nil, errors.New("zero allowed devices") + } + return deviceMap, nil +} + +// rejectByDevice returns a RejectFunc that rejects files which are on a +// different file systems than the files/dirs in samples. +func rejectByDevice(samples []string) (RejectFunc, error) { + allowed, err := gatherDevices(samples) + if err != nil { + return nil, err + } + debug.Log("allowed devices: %v\n", allowed) + + return func(item string, fi os.FileInfo) bool { + if fi == nil { + return false + } + + id, err := fs.DeviceID(fi) + if err != nil { + // This should never happen because gatherDevices() would have + // errored out earlier. If it still does that's a reason to panic. + panic(err) + } + + for dir := item; dir != ""; dir = filepath.Dir(dir) { + debug.Log("item %v, test dir %v", item, dir) + + allowedID, ok := allowed[dir] + if !ok { + continue + } + + if allowedID != id { + debug.Log("path %q on disallowed device %d", item, id) + return true + } + + return false + } + + panic(fmt.Sprintf("item %v, device id %v not found, allowedDevs: %v", item, id, allowed)) + }, nil +} From e5d4e335093ac11c3cb20f9380130b9063969ccb Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 10 Sep 2017 15:14:11 +0200 Subject: [PATCH 08/14] Improve error message if no targets specified --- cmd/restic/cmd_backup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index a994717f4..dbf8cb423 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -324,7 +324,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error { // same time args = append(args, fromfile...) if len(args) == 0 { - return errors.Fatal("wrong number of parameters") + return errors.Fatal("nothing to backup, please specify target files/dirs") } target := make([]string, 0, len(args)) From 2fdca5d310a1238670b85acee8467fb9783ee514 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 10 Sep 2017 15:16:05 +0200 Subject: [PATCH 09/14] Improve debug message --- cmd/restic/exclude.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go index d520f9c80..58aef7f1b 100644 --- a/cmd/restic/exclude.go +++ b/cmd/restic/exclude.go @@ -29,7 +29,7 @@ func rejectByPattern(patterns []string) RejectFunc { } if matched { - debug.Log("path %q excluded by a filter", item) + debug.Log("path %q excluded by an exclude pattern", item) return true } From 47ddd34266f55f9e9282deffda366c6c2c84465f Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 10 Sep 2017 15:20:26 +0200 Subject: [PATCH 10/14] Improve test --- cmd/restic/exclude_test.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cmd/restic/exclude_test.go b/cmd/restic/exclude_test.go index cdbd4ef55..73454f226 100644 --- a/cmd/restic/exclude_test.go +++ b/cmd/restic/exclude_test.go @@ -2,9 +2,10 @@ package main import ( "io/ioutil" - "os" "path/filepath" "testing" + + "github.com/restic/restic/internal/test" ) func TestIsExcludedByFile(t *testing.T) { @@ -29,13 +30,11 @@ func TestIsExcludedByFile(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - tempDir, err := ioutil.TempDir("", "restic-test-") - if err != nil { - t.Fatalf("could not create temp dir: %v", err) - } - defer os.RemoveAll(tempDir) + tempDir, cleanup := test.TempDir(t) + defer cleanup() + foo := filepath.Join(tempDir, "foo") - err = ioutil.WriteFile(foo, []byte("foo"), 0666) + err := ioutil.WriteFile(foo, []byte("foo"), 0666) if err != nil { t.Fatalf("could not write file: %v", err) } From 0b2947dedb524f3fbc40fd75d2152eb82b534214 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 10 Sep 2017 15:31:58 +0200 Subject: [PATCH 11/14] Add test for rejectByPattern --- cmd/restic/exclude_test.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/cmd/restic/exclude_test.go b/cmd/restic/exclude_test.go index 73454f226..ec9b0bade 100644 --- a/cmd/restic/exclude_test.go +++ b/cmd/restic/exclude_test.go @@ -8,6 +8,33 @@ import ( "github.com/restic/restic/internal/test" ) +func TestRejectByPattern(t *testing.T) { + var tests = []struct { + filename string + reject bool + }{ + {filename: "/home/user/foo.go", reject: true}, + {filename: "/home/user/foo.c", reject: false}, + {filename: "/home/user/foobar", reject: false}, + {filename: "/home/user/foobar/x", reject: true}, + {filename: "/home/user/README", reject: false}, + {filename: "/home/user/README.md", reject: true}, + } + + patterns := []string{"*.go", "README.md", "/home/user/foobar/*"} + + for _, tc := range tests { + t.Run("", func(t *testing.T) { + reject := rejectByPattern(patterns) + res := reject(tc.filename, nil) + if res != tc.reject { + t.Fatalf("wrong result for filename %v: want %v, got %v", + tc.filename, tc.reject, res) + } + }) + } +} + func TestIsExcludedByFile(t *testing.T) { const ( tagFilename = "CACHEDIR.TAG" From 89938bc21c7196deaea43f16feef56f598c40806 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 10 Sep 2017 15:33:20 +0200 Subject: [PATCH 12/14] Update manual pages --- doc/man/restic-backup.1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/man/restic-backup.1 b/doc/man/restic-backup.1 index 93e76ca19..090bea4c2 100644 --- a/doc/man/restic-backup.1 +++ b/doc/man/restic-backup.1 @@ -33,8 +33,8 @@ given as the arguments. read exclude patterns from a \fB\fCfile\fR (can be specified multiple times) .PP -\fB\-\-exclude\-if\-present\fP="" - takes filename[:header], exclude contents of directories containing filename (except filename itself) if header of that file is as provided +\fB\-\-exclude\-if\-present\fP=[] + takes filename[:header], exclude contents of directories containing filename (except filename itself) if header of that file is as provided (can be specified multiple times) .PP \fB\-\-files\-from\fP="" From a8aa4eb06c1fc03278add3b2513dbbd890df04d5 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 10 Sep 2017 20:28:21 +0200 Subject: [PATCH 13/14] Rename parameter filename -> path --- cmd/restic/exclude.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go index 58aef7f1b..369c4df9a 100644 --- a/cmd/restic/exclude.go +++ b/cmd/restic/exclude.go @@ -17,7 +17,7 @@ import ( // RejectFunc is a function that takes a filename and os.FileInfo of a // file that would be included in the backup. The function returns true if it // should be excluded (rejected) from the backup. -type RejectFunc func(filename string, fi os.FileInfo) bool +type RejectFunc func(path string, fi os.FileInfo) bool // rejectByPattern returns a RejectFunc which rejects files that match // one of the patterns. From c8e05d1f2a587d3e8466192e8ddff0432a79bd6b Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 10 Sep 2017 20:29:08 +0200 Subject: [PATCH 14/14] Add entry to CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f13ae3d58..70a1355f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,7 @@ Small changes with a specific name (and contents) is present. https://github.com/restic/restic/issues/317 https://github.com/restic/restic/pull/1170 + https://github.com/restic/restic/pull/1224 Important Changes in 0.7.1