rdelaage
/
restic
mirror of https://github.com/restic/restic.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #4705 from MichaelEischer/snapshot-statistics 

Store snapshot statistics & print snapshot size
This commit is contained in:
Michael Eischer 2024-03-28 22:41:45 +01:00 committed by GitHub
parent 71b6284155 e71660cd1e
commit 7f9ad1c3db
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 369 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
changelog/unreleased/issue-693 Normal file
View File

@ -0,0 +1,12 @@
Enhancement: Support printing snapshot size in `snapshots` command
The `snapshots` command now supports printing the snapshot size for snapshots
created using this or a future restic version. For this, the `backup` command
now stores the backup summary statistics in the snapshot.
The text output of the `snapshots` command only shows the snapshot size. The
other statistics are only included in the JSON output. To inspect these
statistics use `restic snapshots --json` or `restic cat snapshot <snapshotID>`.
https://github.com/restic/restic/issues/693
https://github.com/restic/restic/pull/4705

6
cmd/restic/cmd_backup.go
View File

@ -451,6 +451,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
}
timeStamp := time.Now()
backupStart := timeStamp
if opts.TimeStamp != "" {
timeStamp, err = time.ParseInLocation(TimeFormat, opts.TimeStamp, time.Local)
if err != nil {
@ -640,6 +641,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
snapshotOpts := archiver.SnapshotOptions{
Excludes: opts.Excludes,
Tags: opts.Tags.Flatten(),
BackupStart: backupStart,
Time: timeStamp,
Hostname: opts.Host,
ParentSnapshot: parentSnapshot,
@ -649,7 +651,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
if !gopts.JSON {
progressPrinter.V("start backup on %v", targets)
}
_, id, err := arch.Snapshot(ctx, targets, snapshotOpts)
_, id, summary, err := arch.Snapshot(ctx, targets, snapshotOpts)
// cleanly shutdown all running goroutines
cancel()
@ -663,7 +665,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
}
// Report finished execution
progressReporter.Finish(id, opts.DryRun)
progressReporter.Finish(id, summary, opts.DryRun)
if !gopts.JSON && !opts.DryRun {
progressPrinter.P("snapshot %s saved\n", id.Str())
}

14
cmd/restic/cmd_snapshots.go
View File

@ -9,6 +9,7 @@ import (
"strings"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/ui"
"github.com/restic/restic/internal/ui/table"
"github.com/spf13/cobra"
)
@ -163,6 +164,11 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
keepReasons[*id] = reasons[i]
}
}
// check if any snapshot contains a summary
hasSize := false
for _, sn := range list {
hasSize = hasSize || (sn.Summary != nil)
}
// always sort the snapshots so that the newer ones are listed last
sort.SliceStable(list, func(i, j int) bool {
@ -198,6 +204,9 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
tab.AddColumn("Reasons", `{{ join .Reasons "\n" }}`)
}
tab.AddColumn("Paths", `{{ join .Paths "\n" }}`)
if hasSize {
tab.AddColumn("Size", `{{ .Size }}`)
}
}
type snapshot struct {
@ -207,6 +216,7 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
Tags []string
Reasons []string
Paths []string
Size string
}
var multiline bool
@ -228,6 +238,10 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
multiline = true
}
if sn.Summary != nil {
data.Size = ui.FormatBytes(sn.Summary.TotalBytesProcessed)
}
tab.AddRow(data)
}

56
doc/045_working_with_repos.rst
View File

@ -18,19 +18,21 @@ Working with repositories
Listing all snapshots
=====================
Now, you can list all the snapshots stored in the repository:
Now, you can list all the snapshots stored in the repository. The size column
only exists for snapshots created using restic 0.17.0 or later. It reflects the
size of the contained files at the time when the snapshot was created.
.. code-block:: console
$ restic -r /srv/restic-repo snapshots
enter password for repository:
ID Date Host Tags Directory
----------------------------------------------------------------------
40dc1520 2015-05-08 21:38:30 kasimir /home/user/work
79766175 2015-05-08 21:40:19 kasimir /home/user/work
bdbd3439 2015-05-08 21:45:17 luigi /home/art
590c8fc8 2015-05-08 21:47:38 kazik /srv
9f0bc19e 2015-05-08 21:46:11 luigi /srv
ID Date Host Tags Directory Size
-------------------------------------------------------------------------
40dc1520 2015-05-08 21:38:30 kasimir /home/user/work 20.643GiB
79766175 2015-05-08 21:40:19 kasimir /home/user/work 20.645GiB
bdbd3439 2015-05-08 21:45:17 luigi /home/art 3.141GiB
590c8fc8 2015-05-08 21:47:38 kazik /srv 580.200MiB
9f0bc19e 2015-05-08 21:46:11 luigi /srv 572.180MiB
You can filter the listing by directory path:
@ -38,10 +40,10 @@ You can filter the listing by directory path:
$ restic -r /srv/restic-repo snapshots --path="/srv"
enter password for repository:
ID Date Host Tags Directory
----------------------------------------------------------------------
590c8fc8 2015-05-08 21:47:38 kazik /srv
9f0bc19e 2015-05-08 21:46:11 luigi /srv
ID Date Host Tags Directory Size
-------------------------------------------------------------------
590c8fc8 2015-05-08 21:47:38 kazik /srv 580.200MiB
9f0bc19e 2015-05-08 21:46:11 luigi /srv 572.180MiB
Or filter by host:
@ -49,10 +51,10 @@ Or filter by host:
$ restic -r /srv/restic-repo snapshots --host luigi
enter password for repository:
ID Date Host Tags Directory
----------------------------------------------------------------------
bdbd3439 2015-05-08 21:45:17 luigi /home/art
9f0bc19e 2015-05-08 21:46:11 luigi /srv
ID Date Host Tags Directory Size
-------------------------------------------------------------------
bdbd3439 2015-05-08 21:45:17 luigi /home/art 3.141GiB
9f0bc19e 2015-05-08 21:46:11 luigi /srv 572.180MiB
Combining filters is also possible.
@ -64,21 +66,21 @@ Furthermore you can group the output by the same filters (host, paths, tags):
enter password for repository:
snapshots for (host [kasimir])
ID Date Host Tags Directory
----------------------------------------------------------------------
40dc1520 2015-05-08 21:38:30 kasimir /home/user/work
79766175 2015-05-08 21:40:19 kasimir /home/user/work
ID Date Host Tags Directory Size
------------------------------------------------------------------------
40dc1520 2015-05-08 21:38:30 kasimir /home/user/work 20.643GiB
79766175 2015-05-08 21:40:19 kasimir /home/user/work 20.645GiB
2 snapshots
snapshots for (host [luigi])
ID Date Host Tags Directory
----------------------------------------------------------------------
bdbd3439 2015-05-08 21:45:17 luigi /home/art
9f0bc19e 2015-05-08 21:46:11 luigi /srv
ID Date Host Tags Directory Size
-------------------------------------------------------------------
bdbd3439 2015-05-08 21:45:17 luigi /home/art 3.141GiB
9f0bc19e 2015-05-08 21:46:11 luigi /srv 572.180MiB
2 snapshots
snapshots for (host [kazik])
ID Date Host Tags Directory
----------------------------------------------------------------------
590c8fc8 2015-05-08 21:47:38 kazik /srv
ID Date Host Tags Directory Size
-------------------------------------------------------------------
590c8fc8 2015-05-08 21:47:38 kazik /srv 580.200MiB
1 snapshots

41
doc/075_scripting.rst
View File

@ -163,7 +163,9 @@ Summary is the last output line in a successful backup.
+---------------------------+---------------------------------------------------------+
| ``tree_blobs`` | Number of tree blobs |
+---------------------------+---------------------------------------------------------+
| ``data_added`` | Amount of data added, in bytes |
| ``data_added`` | Amount of (uncompressed) data added, in bytes |
+---------------------------+---------------------------------------------------------+
| ``data_added_packed`` | Amount of data added (after compression), in bytes |
+---------------------------+---------------------------------------------------------+
| ``total_files_processed`` | Total number of files processed |
+---------------------------+---------------------------------------------------------+
@ -551,11 +553,48 @@ The snapshots command returns a single JSON object, an array with objects of the
+---------------------+--------------------------------------------------+
| ``program_version`` | restic version used to create snapshot |
+---------------------+--------------------------------------------------+
| ``summary`` | Snapshot statistics, see "Summary object" |
+---------------------+--------------------------------------------------+
| ``id`` | Snapshot ID |
+---------------------+--------------------------------------------------+
| ``short_id`` | Snapshot ID, short form |
+---------------------+--------------------------------------------------+
Summary object
The contained statistics reflect the information at the point in time when the snapshot
was created.
+---------------------------+---------------------------------------------------------+
| ``backup_start`` | Time at which the backup was started |
+---------------------------+---------------------------------------------------------+
| ``backup_end`` | Time at which the backup was completed |
+---------------------------+---------------------------------------------------------+
| ``files_new`` | Number of new files |
+---------------------------+---------------------------------------------------------+
| ``files_changed`` | Number of files that changed |
+---------------------------+---------------------------------------------------------+
| ``files_unmodified`` | Number of files that did not change |
+---------------------------+---------------------------------------------------------+
| ``dirs_new`` | Number of new directories |
+---------------------------+---------------------------------------------------------+
| ``dirs_changed`` | Number of directories that changed |
+---------------------------+---------------------------------------------------------+
| ``dirs_unmodified`` | Number of directories that did not change |
+---------------------------+---------------------------------------------------------+
| ``data_blobs`` | Number of data blobs |
+---------------------------+---------------------------------------------------------+
| ``tree_blobs`` | Number of tree blobs |
+---------------------------+---------------------------------------------------------+
| ``data_added`` | Amount of (uncompressed) data added, in bytes |
+---------------------------+---------------------------------------------------------+
| ``data_added_packed`` | Amount of data added (after compression), in bytes |
+---------------------------+---------------------------------------------------------+
| ``total_files_processed`` | Total number of files processed |
+---------------------------+---------------------------------------------------------+
| ``total_bytes_processed`` | Total number of bytes processed |
+---------------------------+---------------------------------------------------------+
stats
-----

121
internal/archiver/archiver.go
View File

@ -8,6 +8,7 @@ import (
"runtime"
"sort"
"strings"
"sync"
"time"
"github.com/restic/restic/internal/debug"
@ -41,6 +42,18 @@ type ItemStats struct {
TreeSizeInRepo uint64 // sum of the bytes added to the repo (including compression and crypto overhead)
}
type ChangeStats struct {
New uint
Changed uint
Unchanged uint
}
type Summary struct {
Files, Dirs ChangeStats
ProcessedBytes uint64
ItemStats
}
// Add adds other to the current ItemStats.
func (s *ItemStats) Add(other ItemStats) {
s.DataBlobs += other.DataBlobs
@ -62,6 +75,8 @@ type Archiver struct {
blobSaver *BlobSaver
fileSaver *FileSaver
treeSaver *TreeSaver
mu sync.Mutex
summary *Summary
// Error is called for all errors that occur during backup.
Error ErrorFunc
@ -183,6 +198,44 @@ func (arch *Archiver) error(item string, err error) error {
return errf
}
func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s ItemStats, d time.Duration) {
arch.CompleteItem(item, previous, current, s, d)
arch.mu.Lock()
defer arch.mu.Unlock()
arch.summary.ItemStats.Add(s)
if current != nil {
arch.summary.ProcessedBytes += current.Size
} else {
// last item or an error occurred
return
}
switch current.Type {
case "dir":
switch {
case previous == nil:
arch.summary.Dirs.New++
case previous.Equals(*current):
arch.summary.Dirs.Unchanged++
default:
arch.summary.Dirs.Changed++
}
case "file":
switch {
case previous == nil:
arch.summary.Files.New++
case previous.Equals(*current):
arch.summary.Files.Unchanged++
default:
arch.summary.Files.Changed++
}
}
}
// nodeFromFileInfo returns the restic node from an os.FileInfo.
func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo) (*restic.Node, error) {
node, err := restic.NodeFromFileInfo(filename, fi)
@ -231,9 +284,9 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error {
return err
}
// SaveDir stores a directory in the repo and returns the node. snPath is the
// saveDir stores a directory in the repo and returns the node. snPath is the
// path within the current snapshot.
func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete CompleteFunc) (d FutureNode, err error) {
func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete CompleteFunc) (d FutureNode, err error) {
debug.Log("%v %v", snPath, dir)
treeNode, err := arch.nodeFromFileInfo(snPath, dir, fi)
@ -259,7 +312,7 @@ func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi
pathname := arch.FS.Join(dir, name)
oldNode := previous.Find(name)
snItem := join(snPath, name)
fn, excluded, err := arch.Save(ctx, snItem, pathname, oldNode)
fn, excluded, err := arch.save(ctx, snItem, pathname, oldNode)
// return error early if possible
if err != nil {
@ -343,14 +396,14 @@ func (arch *Archiver) allBlobsPresent(previous *restic.Node) bool {
return true
}
// Save saves a target (file or directory) to the repo. If the item is
// save saves a target (file or directory) to the repo. If the item is
// excluded, this function returns a nil node and error, with excluded set to
// true.
//
// Errors and completion needs to be handled by the caller.
//
// snPath is the path within the current snapshot.
func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous *restic.Node) (fn FutureNode, excluded bool, err error) {
func (arch *Archiver) save(ctx context.Context, snPath, target string, previous *restic.Node) (fn FutureNode, excluded bool, err error) {
start := time.Now()
debug.Log("%v target %q, previous %v", snPath, target, previous)
@ -389,7 +442,7 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
if previous != nil && !fileChanged(fi, previous, arch.ChangeIgnoreFlags) {
if arch.allBlobsPresent(previous) {
debug.Log("%v hasn't changed, using old list of blobs", target)
arch.CompleteItem(snPath, previous, previous, ItemStats{}, time.Since(start))
arch.trackItem(snPath, previous, previous, ItemStats{}, time.Since(start))
arch.CompleteBlob(previous.Size)
node, err := arch.nodeFromFileInfo(snPath, target, fi)
if err != nil {
@ -454,9 +507,9 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
fn = arch.fileSaver.Save(ctx, snPath, target, file, fi, func() {
arch.StartFile(snPath)
}, func() {
arch.CompleteItem(snPath, nil, nil, ItemStats{}, 0)
arch.trackItem(snPath, nil, nil, ItemStats{}, 0)
}, func(node *restic.Node, stats ItemStats) {
arch.CompleteItem(snPath, previous, node, stats, time.Since(start))
arch.trackItem(snPath, previous, node, stats, time.Since(start))
})
case fi.IsDir():
@ -471,9 +524,9 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
return FutureNode{}, false, err
}
fn, err = arch.SaveDir(ctx, snPath, target, fi, oldSubtree,
fn, err = arch.saveDir(ctx, snPath, target, fi, oldSubtree,
func(node *restic.Node, stats ItemStats) {
arch.CompleteItem(snItem, previous, node, stats, time.Since(start))
arch.trackItem(snItem, previous, node, stats, time.Since(start))
})
if err != nil {
debug.Log("SaveDir for %v returned error: %v", snPath, err)
@ -554,9 +607,9 @@ func (arch *Archiver) statDir(dir string) (os.FileInfo, error) {
return fi, nil
}
// SaveTree stores a Tree in the repo, returned is the tree. snPath is the path
// saveTree stores a Tree in the repo, returned is the tree. snPath is the path
// within the current snapshot.
func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, previous *restic.Tree, complete CompleteFunc) (FutureNode, int, error) {
func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *Tree, previous *restic.Tree, complete CompleteFunc) (FutureNode, int, error) {
var node *restic.Node
if snPath != "/" {
@ -594,7 +647,7 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree,
// this is a leaf node
if subatree.Leaf() {
fn, excluded, err := arch.Save(ctx, join(snPath, name), subatree.Path, previous.Find(name))
fn, excluded, err := arch.save(ctx, join(snPath, name), subatree.Path, previous.Find(name))
if err != nil {
err = arch.error(subatree.Path, err)
@ -628,8 +681,8 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree,
}
// not a leaf node, archive subtree
fn, _, err := arch.SaveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *restic.Node, is ItemStats) {
arch.CompleteItem(snItem, oldNode, n, is, time.Since(start))
fn, _, err := arch.saveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *restic.Node, is ItemStats) {
arch.trackItem(snItem, oldNode, n, is, time.Since(start))
})
if err != nil {
return FutureNode{}, 0, err
@ -697,6 +750,7 @@ type SnapshotOptions struct {
Tags restic.TagList
Hostname string
Excludes []string
BackupStart time.Time
Time time.Time
ParentSnapshot *restic.Snapshot
ProgramVersion string
@ -747,15 +801,17 @@ func (arch *Archiver) stopWorkers() {
}
// Snapshot saves several targets and returns a snapshot.
func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts SnapshotOptions) (*restic.Snapshot, restic.ID, error) {
func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts SnapshotOptions) (*restic.Snapshot, restic.ID, *Summary, error) {
arch.summary = &Summary{}
cleanTargets, err := resolveRelativeTargets(arch.FS, targets)
if err != nil {
return nil, restic.ID{}, err
return nil, restic.ID{}, nil, err
}
atree, err := NewTree(arch.FS, cleanTargets)
if err != nil {
return nil, restic.ID{}, err
return nil, restic.ID{}, nil, err
}
var rootTreeID restic.ID
@ -771,8 +827,8 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
arch.runWorkers(wgCtx, wg)
debug.Log("starting snapshot")
fn, nodeCount, err := arch.SaveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(_ *restic.Node, is ItemStats) {
arch.CompleteItem("/", nil, nil, is, time.Since(start))
fn, nodeCount, err := arch.saveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(_ *restic.Node, is ItemStats) {
arch.trackItem("/", nil, nil, is, time.Since(start))
})
if err != nil {
return err
@ -808,12 +864,12 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
})
err = wgUp.Wait()
if err != nil {
return nil, restic.ID{}, err
return nil, restic.ID{}, nil, err
}
sn, err := restic.NewSnapshot(targets, opts.Tags, opts.Hostname, opts.Time)
if err != nil {
return nil, restic.ID{}, err
return nil, restic.ID{}, nil, err
}
sn.ProgramVersion = opts.ProgramVersion
@ -822,11 +878,28 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
sn.Parent = opts.ParentSnapshot.ID()
}
sn.Tree = &rootTreeID
sn.Summary = &restic.SnapshotSummary{
BackupStart: opts.BackupStart,
BackupEnd: time.Now(),
FilesNew: arch.summary.Files.New,
FilesChanged: arch.summary.Files.Changed,
FilesUnmodified: arch.summary.Files.Unchanged,
DirsNew: arch.summary.Dirs.New,
DirsChanged: arch.summary.Dirs.Changed,
DirsUnmodified: arch.summary.Dirs.Unchanged,
DataBlobs: arch.summary.ItemStats.DataBlobs,
TreeBlobs: arch.summary.ItemStats.TreeBlobs,
DataAdded: arch.summary.ItemStats.DataSize + arch.summary.ItemStats.TreeSize,
DataAddedPacked: arch.summary.ItemStats.DataSizeInRepo + arch.summary.ItemStats.TreeSizeInRepo,
TotalFilesProcessed: arch.summary.Files.New + arch.summary.Files.Changed + arch.summary.Files.Unchanged,
TotalBytesProcessed: arch.summary.ProcessedBytes,
}
id, err := restic.SaveSnapshot(ctx, arch.Repo, sn)
if err != nil {
return nil, restic.ID{}, err
return nil, restic.ID{}, nil, err
}
return sn, id, nil
return sn, id, arch.summary, nil
}

189
internal/archiver/archiver_test.go
View File

@ -227,8 +227,9 @@ func TestArchiverSave(t *testing.T) {
return err
}
arch.runWorkers(ctx, wg)
arch.summary = &Summary{}
node, excluded, err := arch.Save(ctx, "/", filepath.Join(tempdir, "file"), nil)
node, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "file"), nil)
if err != nil {
t.Fatal(err)
}
@ -304,8 +305,9 @@ func TestArchiverSaveReaderFS(t *testing.T) {
return err
}
arch.runWorkers(ctx, wg)
arch.summary = &Summary{}
node, excluded, err := arch.Save(ctx, "/", filename, nil)
node, excluded, err := arch.save(ctx, "/", filename, nil)
t.Logf("Save returned %v %v", node, err)
if err != nil {
t.Fatal(err)
@ -832,6 +834,7 @@ func TestArchiverSaveDir(t *testing.T) {
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
arch.runWorkers(ctx, wg)
arch.summary = &Summary{}
chdir := tempdir
if test.chdir != "" {
@ -846,7 +849,7 @@ func TestArchiverSaveDir(t *testing.T) {
t.Fatal(err)
}
ft, err := arch.SaveDir(ctx, "/", test.target, fi, nil, nil)
ft, err := arch.saveDir(ctx, "/", test.target, fi, nil, nil)
if err != nil {
t.Fatal(err)
}
@ -913,13 +916,14 @@ func TestArchiverSaveDirIncremental(t *testing.T) {
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
arch.runWorkers(ctx, wg)
arch.summary = &Summary{}
fi, err := fs.Lstat(tempdir)
if err != nil {
t.Fatal(err)
}
ft, err := arch.SaveDir(ctx, "/", tempdir, fi, nil, nil)
ft, err := arch.saveDir(ctx, "/", tempdir, fi, nil, nil)
if err != nil {
t.Fatal(err)
}
@ -983,9 +987,9 @@ func TestArchiverSaveDirIncremental(t *testing.T) {
// bothZeroOrNeither fails the test if only one of exp, act is zero.
func bothZeroOrNeither(tb testing.TB, exp, act uint64) {
tb.Helper()
if (exp == 0 && act != 0) || (exp != 0 && act == 0) {
_, file, line, _ := runtime.Caller(1)
tb.Fatalf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act)
restictest.Equals(tb, exp, act)
}
}
@ -1005,7 +1009,7 @@ func TestArchiverSaveTree(t *testing.T) {
prepare func(t testing.TB)
targets []string
want TestDir
stat ItemStats
stat Summary
}{
{
src: TestDir{
@ -1015,7 +1019,12 @@ func TestArchiverSaveTree(t *testing.T) {
want: TestDir{
"targetfile": TestFile{Content: string("foobar")},
},
stat: ItemStats{1, 6, 32 + 6, 0, 0, 0},
stat: Summary{
ItemStats: ItemStats{1, 6, 32 + 6, 0, 0, 0},
ProcessedBytes: 6,
Files: ChangeStats{1, 0, 0},
Dirs: ChangeStats{0, 0, 0},
},
},
{
src: TestDir{
@ -1027,7 +1036,12 @@ func TestArchiverSaveTree(t *testing.T) {
"targetfile": TestFile{Content: string("foobar")},
"filesymlink": TestSymlink{Target: "targetfile"},
},
stat: ItemStats{1, 6, 32 + 6, 0, 0, 0},
stat: Summary{
ItemStats: ItemStats{1, 6, 32 + 6, 0, 0, 0},
ProcessedBytes: 6,
Files: ChangeStats{1, 0, 0},
Dirs: ChangeStats{0, 0, 0},
},
},
{
src: TestDir{
@ -1047,7 +1061,12 @@ func TestArchiverSaveTree(t *testing.T) {
"symlink": TestSymlink{Target: "subdir"},
},
},
stat: ItemStats{0, 0, 0, 1, 0x154, 0x16a},
stat: Summary{
ItemStats: ItemStats{0, 0, 0, 1, 0x154, 0x16a},
ProcessedBytes: 0,
Files: ChangeStats{0, 0, 0},
Dirs: ChangeStats{1, 0, 0},
},
},
{
src: TestDir{
@ -1071,7 +1090,12 @@ func TestArchiverSaveTree(t *testing.T) {
},
},
},
stat: ItemStats{1, 6, 32 + 6, 3, 0x47f, 0x4c1},
stat: Summary{
ItemStats: ItemStats{1, 6, 32 + 6, 3, 0x47f, 0x4c1},
ProcessedBytes: 6,
Files: ChangeStats{1, 0, 0},
Dirs: ChangeStats{3, 0, 0},
},
},
}
@ -1083,18 +1107,11 @@ func TestArchiverSaveTree(t *testing.T) {
arch := New(repo, testFS, Options{})
var stat ItemStats
lock := &sync.Mutex{}
arch.CompleteItem = func(item string, previous, current *restic.Node, s ItemStats, d time.Duration) {
lock.Lock()
defer lock.Unlock()
stat.Add(s)
}
wg, ctx := errgroup.WithContext(context.TODO())
repo.StartPackUploader(ctx, wg)
arch.runWorkers(ctx, wg)
arch.summary = &Summary{}
back := restictest.Chdir(t, tempdir)
defer back()
@ -1108,7 +1125,7 @@ func TestArchiverSaveTree(t *testing.T) {
t.Fatal(err)
}
fn, _, err := arch.SaveTree(ctx, "/", atree, nil, nil)
fn, _, err := arch.saveTree(ctx, "/", atree, nil, nil)
if err != nil {
t.Fatal(err)
}
@ -1135,11 +1152,15 @@ func TestArchiverSaveTree(t *testing.T) {
want = test.src
}
TestEnsureTree(context.TODO(), t, "/", repo, treeID, want)
stat := arch.summary
bothZeroOrNeither(t, uint64(test.stat.DataBlobs), uint64(stat.DataBlobs))
bothZeroOrNeither(t, uint64(test.stat.TreeBlobs), uint64(stat.TreeBlobs))
bothZeroOrNeither(t, test.stat.DataSize, stat.DataSize)
bothZeroOrNeither(t, test.stat.DataSizeInRepo, stat.DataSizeInRepo)
bothZeroOrNeither(t, test.stat.TreeSizeInRepo, stat.TreeSizeInRepo)
restictest.Equals(t, test.stat.ProcessedBytes, stat.ProcessedBytes)
restictest.Equals(t, test.stat.Files, stat.Files)
restictest.Equals(t, test.stat.Dirs, stat.Dirs)
})
}
}
@ -1396,7 +1417,7 @@ func TestArchiverSnapshot(t *testing.T) {
}
t.Logf("targets: %v", targets)
sn, snapshotID, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()})
sn, snapshotID, _, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()})
if err != nil {
t.Fatal(err)
}
@ -1544,7 +1565,7 @@ func TestArchiverSnapshotSelect(t *testing.T) {
defer back()
targets := []string{"."}
_, snapshotID, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()})
_, snapshotID, _, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()})
if test.err != "" {
if err == nil {
t.Fatalf("expected error not found, got %v, wanted %q", err, test.err)
@ -1617,17 +1638,85 @@ func (f MockFile) Read(p []byte) (int, error) {
return n, err
}
func checkSnapshotStats(t *testing.T, sn *restic.Snapshot, stat Summary) {
restictest.Equals(t, stat.Files.New, sn.Summary.FilesNew)
restictest.Equals(t, stat.Files.Changed, sn.Summary.FilesChanged)
restictest.Equals(t, stat.Files.Unchanged, sn.Summary.FilesUnmodified)
restictest.Equals(t, stat.Dirs.New, sn.Summary.DirsNew)
restictest.Equals(t, stat.Dirs.Changed, sn.Summary.DirsChanged)
restictest.Equals(t, stat.Dirs.Unchanged, sn.Summary.DirsUnmodified)
restictest.Equals(t, stat.ProcessedBytes, sn.Summary.TotalBytesProcessed)
restictest.Equals(t, stat.Files.New+stat.Files.Changed+stat.Files.Unchanged, sn.Summary.TotalFilesProcessed)
bothZeroOrNeither(t, uint64(stat.DataBlobs), uint64(sn.Summary.DataBlobs))
bothZeroOrNeither(t, uint64(stat.TreeBlobs), uint64(sn.Summary.TreeBlobs))
bothZeroOrNeither(t, uint64(stat.DataSize+stat.TreeSize), uint64(sn.Summary.DataAdded))
bothZeroOrNeither(t, uint64(stat.DataSizeInRepo+stat.TreeSizeInRepo), uint64(sn.Summary.DataAddedPacked))
}
func TestArchiverParent(t *testing.T) {
var tests = []struct {
src TestDir
read map[string]int // tracks number of times a file must have been read
src TestDir
modify func(path string)
statInitial Summary
statSecond Summary
}{
{
src: TestDir{
"targetfile": TestFile{Content: string(restictest.Random(888, 2*1024*1024+5000))},
},
read: map[string]int{
"targetfile": 1,
statInitial: Summary{
Files: ChangeStats{1, 0, 0},
Dirs: ChangeStats{0, 0, 0},
ProcessedBytes: 2102152,
ItemStats: ItemStats{3, 0x201593, 0x201632, 1, 0, 0},
},
statSecond: Summary{
Files: ChangeStats{0, 0, 1},
Dirs: ChangeStats{0, 0, 0},
ProcessedBytes: 2102152,
},
},
{
src: TestDir{
"targetDir": TestDir{
"targetfile": TestFile{Content: string(restictest.Random(888, 1234))},
"targetfile2": TestFile{Content: string(restictest.Random(888, 1235))},
},
},
statInitial: Summary{
Files: ChangeStats{2, 0, 0},
Dirs: ChangeStats{1, 0, 0},
ProcessedBytes: 2469,
ItemStats: ItemStats{2, 0xe1c, 0xcd9, 2, 0, 0},
},
statSecond: Summary{
Files: ChangeStats{0, 0, 2},
Dirs: ChangeStats{0, 0, 1},
ProcessedBytes: 2469,
},
},
{
src: TestDir{
"targetDir": TestDir{
"targetfile": TestFile{Content: string(restictest.Random(888, 1234))},
},
"targetfile2": TestFile{Content: string(restictest.Random(888, 1235))},
},
modify: func(path string) {
remove(t, filepath.Join(path, "targetDir", "targetfile"))
save(t, filepath.Join(path, "targetfile2"), []byte("foobar"))
},
statInitial: Summary{
Files: ChangeStats{2, 0, 0},
Dirs: ChangeStats{1, 0, 0},
ProcessedBytes: 2469,
ItemStats: ItemStats{2, 0xe13, 0xcf8, 2, 0, 0},
},
statSecond: Summary{
Files: ChangeStats{0, 1, 0},
Dirs: ChangeStats{0, 1, 0},
ProcessedBytes: 6,
ItemStats: ItemStats{1, 0x305, 0x233, 2, 0, 0},
},
},
}
@ -1649,7 +1738,7 @@ func TestArchiverParent(t *testing.T) {
back := restictest.Chdir(t, tempdir)
defer back()
firstSnapshot, firstSnapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
firstSnapshot, firstSnapshotID, summary, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
if err != nil {
t.Fatal(err)
}
@ -1674,33 +1763,33 @@ func TestArchiverParent(t *testing.T) {
}
return nil
})
restictest.Equals(t, test.statInitial.Files, summary.Files)
restictest.Equals(t, test.statInitial.Dirs, summary.Dirs)
restictest.Equals(t, test.statInitial.ProcessedBytes, summary.ProcessedBytes)
checkSnapshotStats(t, firstSnapshot, test.statInitial)
if test.modify != nil {
test.modify(tempdir)
}
opts := SnapshotOptions{
Time: time.Now(),
ParentSnapshot: firstSnapshot,
}
_, secondSnapshotID, err := arch.Snapshot(ctx, []string{"."}, opts)
testFS.bytesRead = map[string]int{}
secondSnapshot, secondSnapshotID, summary, err := arch.Snapshot(ctx, []string{"."}, opts)
if err != nil {
t.Fatal(err)
}
// check that all files still been read exactly once
TestWalkFiles(t, ".", test.src, func(filename string, item interface{}) error {
file, ok := item.(TestFile)
if !ok {
return nil
}
n, ok := testFS.bytesRead[filename]
if !ok {
t.Fatalf("file %v was not read at all", filename)
}
if n != len(file.Content) {
t.Fatalf("file %v: read %v bytes, wanted %v bytes", filename, n, len(file.Content))
}
return nil
})
if test.modify == nil {
// check that no files were read this time
restictest.Equals(t, map[string]int{}, testFS.bytesRead)
}
restictest.Equals(t, test.statSecond.Files, summary.Files)
restictest.Equals(t, test.statSecond.Dirs, summary.Dirs)
restictest.Equals(t, test.statSecond.ProcessedBytes, summary.ProcessedBytes)
checkSnapshotStats(t, secondSnapshot, test.statSecond)
t.Logf("second backup saved as %v", secondSnapshotID.Str())
t.Logf("testfs: %v", testFS)
@ -1815,7 +1904,7 @@ func TestArchiverErrorReporting(t *testing.T) {
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
arch.Error = test.errFn
_, snapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
_, snapshotID, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
if test.mustError {
if err != nil {
t.Logf("found expected error (%v), skipping further checks", err)
@ -1888,7 +1977,7 @@ func TestArchiverContextCanceled(t *testing.T) {
arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
_, snapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
_, snapshotID, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
if err != nil {
t.Logf("found expected error (%v)", err)
@ -2027,7 +2116,7 @@ func TestArchiverAbortEarlyOnError(t *testing.T) {
SaveBlobConcurrency: 1,
})
_, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
_, _, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
if !errors.Is(err, test.err) {
t.Errorf("expected error (%v) not found, got %v", test.err, err)
}
@ -2055,7 +2144,7 @@ func snapshot(t testing.TB, repo restic.Repository, fs fs.FS, parent *restic.Sna
Time: time.Now(),
ParentSnapshot: parent,
}
snapshot, _, err := arch.Snapshot(ctx, []string{filename}, sopts)
snapshot, _, _, err := arch.Snapshot(ctx, []string{filename}, sopts)
if err != nil {
t.Fatal(err)
}
@ -2240,7 +2329,7 @@ func TestRacyFileSwap(t *testing.T) {
arch.runWorkers(ctx, wg)
// fs.Track will panic if the file was not closed
_, excluded, err := arch.Save(ctx, "/", tempfile, nil)
_, excluded, err := arch.save(ctx, "/", tempfile, nil)
if err == nil {
t.Errorf("Save() should have failed")
}

2
internal/archiver/testing.go
View File

@ -32,7 +32,7 @@ func TestSnapshot(t testing.TB, repo restic.Repository, path string, parent *res
}
opts.ParentSnapshot = sn
}
sn, _, err := arch.Snapshot(context.TODO(), []string{path}, opts)
sn, _, _, err := arch.Snapshot(context.TODO(), []string{path}, opts)
if err != nil {
t.Fatal(err)
}

2
internal/archiver/testing_test.go
View File

@ -473,7 +473,7 @@ func TestTestEnsureSnapshot(t *testing.T) {
Hostname: "localhost",
Tags: []string{"test"},
}
_, id, err := arch.Snapshot(ctx, []string{"."}, opts)
_, id, _, err := arch.Snapshot(ctx, []string{"."}, opts)
if err != nil {
t.Fatal(err)
}

2
internal/dump/common_test.go
View File

@ -78,7 +78,7 @@ func WriteTest(t *testing.T, format string, cd CheckDump) {
back := rtest.Chdir(t, tmpdir)
defer back()
sn, _, err := arch.Snapshot(ctx, []string{"."}, archiver.SnapshotOptions{})
sn, _, _, err := arch.Snapshot(ctx, []string{"."}, archiver.SnapshotOptions{})
rtest.OK(t, err)
tree, err := restic.LoadTree(ctx, repo, *sn.Tree)

22
internal/restic/snapshot.go
View File

@ -25,11 +25,31 @@ type Snapshot struct {
Tags []string `json:"tags,omitempty"`
Original *ID `json:"original,omitempty"`
ProgramVersion string `json:"program_version,omitempty"`
ProgramVersion string `json:"program_version,omitempty"`
Summary *SnapshotSummary `json:"summary,omitempty"`
id *ID // plaintext ID, used during restore
}
type SnapshotSummary struct {
BackupStart time.Time `json:"backup_start"`
BackupEnd time.Time `json:"backup_end"`
// statistics from the backup json output
FilesNew uint `json:"files_new"`
FilesChanged uint `json:"files_changed"`
FilesUnmodified uint `json:"files_unmodified"`
DirsNew uint `json:"dirs_new"`
DirsChanged uint `json:"dirs_changed"`
DirsUnmodified uint `json:"dirs_unmodified"`
DataBlobs int `json:"data_blobs"`
TreeBlobs int `json:"tree_blobs"`
DataAdded uint64 `json:"data_added"`
DataAddedPacked uint64 `json:"data_added_packed"`
TotalFilesProcessed uint `json:"total_files_processed"`
TotalBytesProcessed uint64 `json:"total_bytes_processed"`
}
// NewSnapshot returns an initialized snapshot struct for the current user and
// time.
func NewSnapshot(paths []string, tags []string, hostname string, time time.Time) (*Snapshot, error) {

2
internal/restorer/restorer_test.go
View File

@ -858,7 +858,7 @@ func TestRestorerSparseFiles(t *testing.T) {
rtest.OK(t, err)
arch := archiver.New(repo, target, archiver.Options{})
sn, _, err := arch.Snapshot(context.Background(), []string{"/zeros"},
sn, _, _, err := arch.Snapshot(context.Background(), []string{"/zeros"},
archiver.SnapshotOptions{})
rtest.OK(t, err)

4
internal/ui/backup/json.go
View File

@ -163,7 +163,7 @@ func (b *JSONProgress) ReportTotal(start time.Time, s archiver.ScanStats) {
}
// Finish prints the finishing messages.
func (b *JSONProgress) Finish(snapshotID restic.ID, start time.Time, summary *Summary, dryRun bool) {
func (b *JSONProgress) Finish(snapshotID restic.ID, start time.Time, summary *archiver.Summary, dryRun bool) {
b.print(summaryOutput{
MessageType: "summary",
FilesNew: summary.Files.New,
@ -175,6 +175,7 @@ func (b *JSONProgress) Finish(snapshotID restic.ID, start time.Time, summary *Su
DataBlobs: summary.ItemStats.DataBlobs,
TreeBlobs: summary.ItemStats.TreeBlobs,
DataAdded: summary.ItemStats.DataSize + summary.ItemStats.TreeSize,
DataAddedPacked: summary.ItemStats.DataSizeInRepo + summary.ItemStats.TreeSizeInRepo,
TotalFilesProcessed: summary.Files.New + summary.Files.Changed + summary.Files.Unchanged,
TotalBytesProcessed: summary.ProcessedBytes,
TotalDuration: time.Since(start).Seconds(),
@ -230,6 +231,7 @@ type summaryOutput struct {
DataBlobs int `json:"data_blobs"`
TreeBlobs int `json:"tree_blobs"`
DataAdded uint64 `json:"data_added"`
DataAddedPacked uint64 `json:"data_added_packed"`
TotalFilesProcessed uint `json:"total_files_processed"`
TotalBytesProcessed uint64 `json:"total_bytes_processed"`
TotalDuration float64 `json:"total_duration"` // in seconds

49
internal/ui/backup/progress.go
View File

@ -17,7 +17,7 @@ type ProgressPrinter interface {
ScannerError(item string, err error) error
CompleteItem(messageType string, item string, s archiver.ItemStats, d time.Duration)
ReportTotal(start time.Time, s archiver.ScanStats)
Finish(snapshotID restic.ID, start time.Time, summary *Summary, dryRun bool)
Finish(snapshotID restic.ID, start time.Time, summary *archiver.Summary, dryRun bool)
Reset()
P(msg string, args ...interface{})
@ -28,16 +28,6 @@ type Counter struct {
Files, Dirs, Bytes uint64
}
type Summary struct {
Files, Dirs struct {
New uint
Changed uint
Unchanged uint
}
ProcessedBytes uint64
archiver.ItemStats
}
// Progress reports progress for the `backup` command.
type Progress struct {
progress.Updater
@ -52,7 +42,6 @@ type Progress struct {
processed, total Counter
errors uint
summary Summary
printer ProgressPrinter
}
@ -126,16 +115,6 @@ func (p *Progress) CompleteBlob(bytes uint64) {
// CompleteItem is the status callback function for the archiver when a
// file/dir has been saved successfully.
func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s archiver.ItemStats, d time.Duration) {
p.mu.Lock()
p.summary.ItemStats.Add(s)
// for the last item "/", current is nil
if current != nil {
p.summary.ProcessedBytes += current.Size
}
p.mu.Unlock()
if current == nil {
// error occurred, tell the status display to remove the line
p.mu.Lock()
@ -153,21 +132,10 @@ func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s a
switch {
case previous == nil:
p.printer.CompleteItem("dir new", item, s, d)
p.mu.Lock()
p.summary.Dirs.New++
p.mu.Unlock()
case previous.Equals(*current):
p.printer.CompleteItem("dir unchanged", item, s, d)
p.mu.Lock()
p.summary.Dirs.Unchanged++
p.mu.Unlock()
default:
p.printer.CompleteItem("dir modified", item, s, d)
p.mu.Lock()
p.summary.Dirs.Changed++
p.mu.Unlock()
}
case "file":
@ -179,21 +147,10 @@ func (p *Progress) CompleteItem(item string, previous, current *restic.Node, s a
switch {
case previous == nil:
p.printer.CompleteItem("file new", item, s, d)
p.mu.Lock()
p.summary.Files.New++
p.mu.Unlock()
case previous.Equals(*current):
p.printer.CompleteItem("file unchanged", item, s, d)
p.mu.Lock()
p.summary.Files.Unchanged++
p.mu.Unlock()
default:
p.printer.CompleteItem("file modified", item, s, d)
p.mu.Lock()
p.summary.Files.Changed++
p.mu.Unlock()
}
}
}
@ -213,8 +170,8 @@ func (p *Progress) ReportTotal(item string, s archiver.ScanStats) {
}
// Finish prints the finishing messages.
func (p *Progress) Finish(snapshotID restic.ID, dryrun bool) {
func (p *Progress) Finish(snapshotID restic.ID, summary *archiver.Summary, dryrun bool) {
// wait for the status update goroutine to shut down
p.Updater.Done()
p.printer.Finish(snapshotID, p.start, &p.summary, dryrun)
p.printer.Finish(snapshotID, p.start, summary, dryrun)
}

5
internal/ui/backup/progress_test.go
View File

@ -33,11 +33,10 @@ func (p *mockPrinter) CompleteItem(messageType string, _ string, _ archiver.Item
}
func (p *mockPrinter) ReportTotal(_ time.Time, _ archiver.ScanStats) {}
func (p *mockPrinter) Finish(id restic.ID, _ time.Time, summary *Summary, _ bool) {
func (p *mockPrinter) Finish(id restic.ID, _ time.Time, _ *archiver.Summary, _ bool) {
p.Lock()
defer p.Unlock()
_ = *summary // Should not be nil.
p.id = id
}
@ -64,7 +63,7 @@ func TestProgress(t *testing.T) {
time.Sleep(10 * time.Millisecond)
id := restic.NewRandomID()
prog.Finish(id, false)
prog.Finish(id, nil, false)
if !prnt.dirUnchanged {
t.Error(`"dir unchanged" event not seen`)

2
internal/ui/backup/text.go
View File

@ -126,7 +126,7 @@ func (b *TextProgress) Reset() {
}
// Finish prints the finishing messages.
func (b *TextProgress) Finish(_ restic.ID, start time.Time, summary *Summary, dryRun bool) {
func (b *TextProgress) Finish(_ restic.ID, start time.Time, summary *archiver.Summary, dryRun bool) {
b.P("\n")
b.P("Files: %5d new, %5d changed, %5d unmodified\n", summary.Files.New, summary.Files.Changed, summary.Files.Unchanged)
b.P("Dirs: %5d new, %5d changed, %5d unmodified\n", summary.Dirs.New, summary.Dirs.Changed, summary.Dirs.Unchanged)