Merge pull request #3300 from aawsome/backup-dryrun

backup: add --dry-run/-n flag
This commit is contained in:
MichaelEischer 2021-08-04 21:50:38 +02:00 committed by GitHub
commit bc97a3d1f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 334 additions and 2 deletions

View File

@ -0,0 +1,17 @@
Enhancement: Add --dry-run/-n option to backup command
Testing exclude filters and other configuration options required running a
normal backup. Wrong filters could then cause files to be uploaded
unexpectedly. It was also not possible to approximately determine beforehand
how much data has to be uploaded.
We added a new --dry-run/-n option to the backup command, which performs
all the normal steps of a backup without actually writing any changes to
the repository. Passing -vv will log information about files that would
be added, allowing verification of source and exclusion backup options
without committing changes to the repository.
https://github.com/restic/restic/issues/1542
https://github.com/restic/restic/pull/2308
https://github.com/restic/restic/pull/3210
https://github.com/restic/restic/pull/3300

View File

@ -92,6 +92,7 @@ type BackupOptions struct {
IgnoreInode bool IgnoreInode bool
IgnoreCtime bool IgnoreCtime bool
UseFsSnapshot bool UseFsSnapshot bool
DryRun bool
} }
var backupOptions BackupOptions var backupOptions BackupOptions
@ -132,6 +133,7 @@ func init() {
f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories") f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files") f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files")
f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files") f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done")
if runtime.GOOS == "windows" { if runtime.GOOS == "windows" {
f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)") f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
} }
@ -535,6 +537,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
Run(ctx context.Context) error Run(ctx context.Context) error
Error(item string, fi os.FileInfo, err error) error Error(item string, fi os.FileInfo, err error) error
Finish(snapshotID restic.ID) Finish(snapshotID restic.ID)
SetDryRun()
// ui.StdioWrapper // ui.StdioWrapper
Stdout() io.WriteCloser Stdout() io.WriteCloser
@ -554,6 +557,11 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
p = ui.NewBackup(term, gopts.verbosity) p = ui.NewBackup(term, gopts.verbosity)
} }
if opts.DryRun {
repo.SetDryRun()
p.SetDryRun()
}
// use the terminal for stdout/stderr // use the terminal for stdout/stderr
prevStdout, prevStderr := gopts.stdout, gopts.stderr prevStdout, prevStderr := gopts.stdout, gopts.stderr
defer func() { defer func() {
@ -722,7 +730,7 @@ func runBackup(opts BackupOptions, gopts GlobalOptions, term *termstatus.Termina
// Report finished execution // Report finished execution
p.Finish(id) p.Finish(id)
if !gopts.JSON { if !gopts.JSON && !opts.DryRun {
p.P("snapshot %s saved\n", id.Str()) p.P("snapshot %s saved\n", id.Str())
} }
if !success { if !success {

View File

@ -345,6 +345,57 @@ func testBackup(t *testing.T, useFsSnapshot bool) {
testRunCheck(t, env.gopts) testRunCheck(t, env.gopts)
} }
func TestDryRunBackup(t *testing.T) {
env, cleanup := withTestEnvironment(t)
defer cleanup()
testSetupBackupData(t, env)
opts := BackupOptions{}
dryOpts := BackupOptions{DryRun: true}
// dry run before first backup
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
snapshotIDs := testRunList(t, "snapshots", env.gopts)
rtest.Assert(t, len(snapshotIDs) == 0,
"expected no snapshot, got %v", snapshotIDs)
packIDs := testRunList(t, "packs", env.gopts)
rtest.Assert(t, len(packIDs) == 0,
"expected no data, got %v", snapshotIDs)
indexIDs := testRunList(t, "index", env.gopts)
rtest.Assert(t, len(indexIDs) == 0,
"expected no index, got %v", snapshotIDs)
// first backup
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
snapshotIDs = testRunList(t, "snapshots", env.gopts)
packIDs = testRunList(t, "packs", env.gopts)
indexIDs = testRunList(t, "index", env.gopts)
// dry run between backups
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
snapshotIDsAfter := testRunList(t, "snapshots", env.gopts)
rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
dataIDsAfter := testRunList(t, "packs", env.gopts)
rtest.Equals(t, packIDs, dataIDsAfter)
indexIDsAfter := testRunList(t, "index", env.gopts)
rtest.Equals(t, indexIDs, indexIDsAfter)
// second backup, implicit incremental
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
snapshotIDs = testRunList(t, "snapshots", env.gopts)
packIDs = testRunList(t, "packs", env.gopts)
indexIDs = testRunList(t, "index", env.gopts)
// another dry run
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, dryOpts, env.gopts)
snapshotIDsAfter = testRunList(t, "snapshots", env.gopts)
rtest.Equals(t, snapshotIDs, snapshotIDsAfter)
dataIDsAfter = testRunList(t, "packs", env.gopts)
rtest.Equals(t, packIDs, dataIDsAfter)
indexIDsAfter = testRunList(t, "index", env.gopts)
rtest.Equals(t, indexIDs, indexIDsAfter)
}
func TestBackupNonExistingFile(t *testing.T) { func TestBackupNonExistingFile(t *testing.T) {
env, cleanup := withTestEnvironment(t) env, cleanup := withTestEnvironment(t)
defer cleanup() defer cleanup()

View File

@ -187,6 +187,23 @@ On **Windows**, a file is considered unchanged when its path, size
and modification time match, and only ``--force`` has any effect. and modification time match, and only ``--force`` has any effect.
The other options are recognized but ignored. The other options are recognized but ignored.
Dry Runs
********
You can perform a backup in dry run mode to see what would happen without
modifying the repo.
- ``--dry-run``/``-n`` Report what would be done, without writing to the repository
Combined with ``--verbose``, you can see a list of changes:
.. code-block:: console
$ restic -r /srv/restic-repo backup ~/work --dry-run -vv | grep "added"
modified /plan.txt, saved in 0.000s (9.110 KiB added)
modified /archive.tar.gz, saved in 0.140s (25.542 MiB added)
Would be added to the repo: 25.551 MiB
Excluding Files Excluding Files
*************** ***************

View File

@ -0,0 +1,79 @@
package dryrun
import (
"context"
"io"
"github.com/restic/restic/internal/debug"
"github.com/restic/restic/internal/restic"
)
// Backend passes reads through to an underlying layer and accepts writes, but
// doesn't do anything. Also removes are ignored.
// So in fact, this backend silently ignores all operations that would modify
// the repo and does normal operations else.
// This is used for `backup --dry-run`.
type Backend struct {
b restic.Backend
}
// statically ensure that RetryBackend implements restic.Backend.
var _ restic.Backend = &Backend{}
// New returns a new backend that saves all data in a map in memory.
func New(be restic.Backend) *Backend {
b := &Backend{b: be}
debug.Log("created new dry backend")
return b
}
// Save adds new Data to the backend.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
if err := h.Valid(); err != nil {
return err
}
debug.Log("faked saving %v bytes at %v", rd.Length(), h)
// don't save anything, just return ok
return nil
}
// Remove deletes a file from the backend.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
return nil
}
// Location returns the location of the backend.
func (be *Backend) Location() string {
return "DRY:" + be.b.Location()
}
// Delete removes all data in the backend.
func (be *Backend) Delete(ctx context.Context) error {
return nil
}
func (be *Backend) Close() error {
return be.b.Close()
}
func (be *Backend) IsNotExist(err error) bool {
return be.b.IsNotExist(err)
}
func (be *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
return be.b.List(ctx, t, fn)
}
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(io.Reader) error) error {
return be.b.Load(ctx, h, length, offset, fn)
}
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
return be.b.Stat(ctx, h)
}
func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
return be.b.Test(ctx, h)
}

View File

@ -0,0 +1,137 @@
package dryrun_test
import (
"context"
"fmt"
"io"
"io/ioutil"
"sort"
"strings"
"testing"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/backend/dryrun"
"github.com/restic/restic/internal/backend/mem"
)
// make sure that Backend implements backend.Backend
var _ restic.Backend = &dryrun.Backend{}
func newBackends() (*dryrun.Backend, restic.Backend) {
m := mem.New()
return dryrun.New(m), m
}
func TestDry(t *testing.T) {
ctx := context.TODO()
d, m := newBackends()
// Since the dry backend is a mostly write-only overlay, the standard backend test suite
// won't pass. Instead, perform a series of operations over the backend, testing the state
// at each step.
steps := []struct {
be restic.Backend
op string
fname string
content string
wantErr string
}{
{d, "loc", "", "DRY:RAM", ""},
{d, "delete", "", "", ""},
{d, "stat", "a", "", "not found"},
{d, "list", "", "", ""},
{d, "save", "", "", "invalid"},
{d, "test", "a", "", ""},
{m, "save", "a", "baz", ""}, // save a directly to the mem backend
{d, "save", "b", "foob", ""}, // b is not saved
{d, "save", "b", "xxx", ""}, // no error as b is not saved
{d, "test", "a", "1", ""},
{d, "test", "b", "", ""},
{d, "stat", "", "", "invalid"},
{d, "stat", "a", "a 3", ""},
{d, "load", "a", "baz", ""},
{d, "load", "b", "", "not found"},
{d, "list", "", "a", ""},
{d, "remove", "c", "", ""},
{d, "stat", "b", "", "not found"},
{d, "list", "", "a", ""},
{d, "remove", "a", "", ""}, // a is in fact not removed
{d, "list", "", "a", ""},
{m, "remove", "a", "", ""}, // remove a from the mem backend
{d, "list", "", "", ""},
{d, "close", "", "", ""},
{d, "close", "", "", ""},
}
for i, step := range steps {
var err error
var boolRes bool
handle := restic.Handle{Type: restic.PackFile, Name: step.fname}
switch step.op {
case "save":
err = step.be.Save(ctx, handle, restic.NewByteReader([]byte(step.content)))
case "test":
boolRes, err = step.be.Test(ctx, handle)
if boolRes != (step.content != "") {
t.Errorf("%d. Test(%q) = %v, want %v", i, step.fname, boolRes, step.content != "")
}
case "list":
fileList := []string{}
err = step.be.List(ctx, restic.PackFile, func(fi restic.FileInfo) error {
fileList = append(fileList, fi.Name)
return nil
})
sort.Strings(fileList)
files := strings.Join(fileList, " ")
if files != step.content {
t.Errorf("%d. List = %q, want %q", i, files, step.content)
}
case "loc":
loc := step.be.Location()
if loc != step.content {
t.Errorf("%d. Location = %q, want %q", i, loc, step.content)
}
case "delete":
err = step.be.Delete(ctx)
case "remove":
err = step.be.Remove(ctx, handle)
case "stat":
var fi restic.FileInfo
fi, err = step.be.Stat(ctx, handle)
if err == nil {
fis := fmt.Sprintf("%s %d", fi.Name, fi.Size)
if fis != step.content {
t.Errorf("%d. Stat = %q, want %q", i, fis, step.content)
}
}
case "load":
data := ""
err = step.be.Load(ctx, handle, 100, 0, func(rd io.Reader) error {
buf, err := ioutil.ReadAll(rd)
data = string(buf)
return err
})
if data != step.content {
t.Errorf("%d. Load = %q, want %q", i, data, step.content)
}
case "close":
err = step.be.Close()
default:
t.Fatalf("%d. unknown step operation %q", i, step.op)
}
if step.wantErr != "" {
if err == nil {
t.Errorf("%d. %s error = nil, want %q", i, step.op, step.wantErr)
} else if !strings.Contains(err.Error(), step.wantErr) {
t.Errorf("%d. %s error = %q, doesn't contain %q", i, step.op, err, step.wantErr)
} else if step.wantErr == "not found" && !step.be.IsNotExist(err) {
t.Errorf("%d. IsNotExist(%s error) = false, want true", i, step.op)
}
} else if err != nil {
t.Errorf("%d. %s error = %q, want nil", i, step.op, err)
}
}
}

View File

@ -10,6 +10,7 @@ import (
"sync" "sync"
"github.com/restic/chunker" "github.com/restic/chunker"
"github.com/restic/restic/internal/backend/dryrun"
"github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/cache"
"github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/debug"
@ -72,6 +73,11 @@ func (r *Repository) UseCache(c *cache.Cache) {
r.be = c.Wrap(r.be) r.be = c.Wrap(r.be)
} }
// SetDryRun sets the repo backend into dry-run mode.
func (r *Repository) SetDryRun() {
r.be = dryrun.New(r.be)
}
// PrefixLength returns the number of bytes required so that all prefixes of // PrefixLength returns the number of bytes required so that all prefixes of
// all IDs of type t are unique. // all IDs of type t are unique.
func (r *Repository) PrefixLength(ctx context.Context, t restic.FileType) (int, error) { func (r *Repository) PrefixLength(ctx context.Context, t restic.FileType) (int, error) {

View File

@ -35,6 +35,7 @@ type Backup struct {
start time.Time start time.Time
totalBytes uint64 totalBytes uint64
dry bool // true if writes are faked
totalCh chan counter totalCh chan counter
processedCh chan counter processedCh chan counter
@ -385,7 +386,11 @@ func (b *Backup) Finish(snapshotID restic.ID) {
b.P("Dirs: %5d new, %5d changed, %5d unmodified\n", b.summary.Dirs.New, b.summary.Dirs.Changed, b.summary.Dirs.Unchanged) b.P("Dirs: %5d new, %5d changed, %5d unmodified\n", b.summary.Dirs.New, b.summary.Dirs.Changed, b.summary.Dirs.Unchanged)
b.V("Data Blobs: %5d new\n", b.summary.ItemStats.DataBlobs) b.V("Data Blobs: %5d new\n", b.summary.ItemStats.DataBlobs)
b.V("Tree Blobs: %5d new\n", b.summary.ItemStats.TreeBlobs) b.V("Tree Blobs: %5d new\n", b.summary.ItemStats.TreeBlobs)
b.P("Added to the repo: %-5s\n", formatBytes(b.summary.ItemStats.DataSize+b.summary.ItemStats.TreeSize)) verb := "Added"
if b.dry {
verb = "Would add"
}
b.P("%s to the repo: %-5s\n", verb, formatBytes(b.summary.ItemStats.DataSize+b.summary.ItemStats.TreeSize))
b.P("\n") b.P("\n")
b.P("processed %v files, %v in %s", b.P("processed %v files, %v in %s",
b.summary.Files.New+b.summary.Files.Changed+b.summary.Files.Unchanged, b.summary.Files.New+b.summary.Files.Changed+b.summary.Files.Unchanged,
@ -399,3 +404,7 @@ func (b *Backup) Finish(snapshotID restic.ID) {
func (b *Backup) SetMinUpdatePause(d time.Duration) { func (b *Backup) SetMinUpdatePause(d time.Duration) {
b.MinUpdatePause = d b.MinUpdatePause = d
} }
func (b *Backup) SetDryRun() {
b.dry = true
}

View File

@ -34,6 +34,7 @@ type Backup struct {
term *termstatus.Terminal term *termstatus.Terminal
v uint v uint
start time.Time start time.Time
dry bool
totalBytes uint64 totalBytes uint64
@ -403,6 +404,7 @@ func (b *Backup) Finish(snapshotID restic.ID) {
TotalBytesProcessed: b.summary.ProcessedBytes, TotalBytesProcessed: b.summary.ProcessedBytes,
TotalDuration: time.Since(b.start).Seconds(), TotalDuration: time.Since(b.start).Seconds(),
SnapshotID: snapshotID.Str(), SnapshotID: snapshotID.Str(),
DryRun: b.dry,
}) })
} }
@ -412,6 +414,11 @@ func (b *Backup) SetMinUpdatePause(d time.Duration) {
b.MinUpdatePause = d b.MinUpdatePause = d
} }
// SetDryRun marks the backup as a "dry run".
func (b *Backup) SetDryRun() {
b.dry = true
}
type statusUpdate struct { type statusUpdate struct {
MessageType string `json:"message_type"` // "status" MessageType string `json:"message_type"` // "status"
SecondsElapsed uint64 `json:"seconds_elapsed,omitempty"` SecondsElapsed uint64 `json:"seconds_elapsed,omitempty"`
@ -457,4 +464,5 @@ type summaryOutput struct {
TotalBytesProcessed uint64 `json:"total_bytes_processed"` TotalBytesProcessed uint64 `json:"total_bytes_processed"`
TotalDuration float64 `json:"total_duration"` // in seconds TotalDuration float64 `json:"total_duration"` // in seconds
SnapshotID string `json:"snapshot_id"` SnapshotID string `json:"snapshot_id"`
DryRun bool `json:"dry_run,omitempty"`
} }