From 77c850148a93b0d659be994fafae078b88ac9305 Mon Sep 17 00:00:00 2001 From: David Vogt Date: Sun, 23 Jan 2022 23:12:29 +0100 Subject: [PATCH 1/3] Add json output for diff command --- changelog/unreleased/issue-2508 | 8 +++ cmd/restic/cmd_diff.go | 111 ++++++++++++++++++++++---------- 2 files changed, 84 insertions(+), 35 deletions(-) create mode 100644 changelog/unreleased/issue-2508 diff --git a/changelog/unreleased/issue-2508 b/changelog/unreleased/issue-2508 new file mode 100644 index 000000000..5adaddd9a --- /dev/null +++ b/changelog/unreleased/issue-2508 @@ -0,0 +1,8 @@ +Enhancement: Support JSON output for diff + +We've added support for getting machine-readable output for snapshot diff, just pass the +flag `--json` for `restic diff` and restic will output a JSON-encoded diff stats and change +list. + +https://github.com/restic/restic/issues/2508 +https://github.com/restic/restic/pull/3592 \ No newline at end of file diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index 9030669d4..9cdd022fd 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "path" "reflect" "sort" @@ -62,15 +63,29 @@ func loadSnapshot(ctx context.Context, repo *repository.Repository, desc string) // Comparer collects all things needed to compare two snapshots. type Comparer struct { - repo restic.Repository - opts DiffOptions + repo restic.Repository + opts DiffOptions + printChange func(change *Change) +} + +type Change struct { + MessageType string `json:"message_type"` // "change" + Path string `json:"path"` + Modifier string `json:"modifier"` +} + +func NewChange(path string, mode string) *Change { + return &Change{MessageType: "change", Path: path, Modifier: mode} } // DiffStat collects stats for all types of items. type DiffStat struct { - Files, Dirs, Others int - DataBlobs, TreeBlobs int - Bytes uint64 + Files int `json:"files"` + Dirs int `json:"dirs"` + Others int `json:"others"` + DataBlobs int `json:"data_blobs"` + TreeBlobs int `json:"tree_blobs"` + Bytes uint64 `json:"bytes"` } // Add adds stats information for node to s. @@ -113,21 +128,14 @@ func addBlobs(bs restic.BlobSet, node *restic.Node) { } } -// DiffStats collects the differences between two snapshots. -type DiffStats struct { - ChangedFiles int - Added DiffStat - Removed DiffStat - BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet -} - -// NewDiffStats creates new stats for a diff run. -func NewDiffStats() *DiffStats { - return &DiffStats{ - BlobsBefore: restic.NewBlobSet(), - BlobsAfter: restic.NewBlobSet(), - BlobsCommon: restic.NewBlobSet(), - } +type DiffStatsContainer struct { + MessageType string `json:"message_type"` // "statistics" + SourceSnapshot string `json:"source_snapshot"` + TargetSnapshot string `json:"target_snapshot"` + ChangedFiles int `json:"changed_files"` + Added DiffStat `json:"added"` + Removed DiffStat `json:"removed"` + BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet `json:"-"` } // updateBlobs updates the blob counters in the stats struct. @@ -162,7 +170,7 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b if node.Type == "dir" { name += "/" } - Printf("%-5s%v\n", mode, name) + c.printChange(NewChange(name, "+")) stats.Add(node) addBlobs(blobs, node) @@ -221,7 +229,7 @@ func uniqueNodeNames(tree1, tree2 *restic.Tree) (tree1Nodes, tree2Nodes map[stri return tree1Nodes, tree2Nodes, uniqueNames } -func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string, id1, id2 restic.ID) error { +func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, prefix string, id1, id2 restic.ID) error { debug.Log("diffing %v to %v", id1, id2) tree1, err := c.repo.LoadTree(ctx, id1) if err != nil { @@ -265,7 +273,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string } if mod != "" { - Printf("%-5s%v\n", mod, name) + c.printChange(NewChange(name, mod)) } if node1.Type == "dir" && node2.Type == "dir" { @@ -284,7 +292,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string if node1.Type == "dir" { prefix += "/" } - Printf("%-5s%v\n", "-", prefix) + c.printChange(NewChange(prefix, "-")) stats.Removed.Add(node1) if node1.Type == "dir" { @@ -298,7 +306,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string if node2.Type == "dir" { prefix += "/" } - Printf("%-5s%v\n", "+", prefix) + c.printChange(NewChange(prefix, "+")) stats.Added.Add(node2) if node2.Type == "dir" { @@ -348,7 +356,9 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error { return err } - Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str()) + if !gopts.JSON { + Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str()) + } if sn1.Tree == nil { return errors.Errorf("snapshot %v has nil tree", sn1.ID().Str()) @@ -361,9 +371,33 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error { c := &Comparer{ repo: repo, opts: diffOptions, + printChange: func(change *Change) { + Printf("%-5s%v\n", change.Modifier, change.Path) + }, } - stats := NewDiffStats() + if gopts.JSON { + enc := json.NewEncoder(gopts.stdout) + c.printChange = func(change *Change) { + err := enc.Encode(change) + if err != nil { + Warnf("JSON encode failed: %v\n", err) + } + } + } + + if gopts.Quiet { + c.printChange = func(change *Change) {} + } + + stats := &DiffStatsContainer{ + MessageType: "statistics", + SourceSnapshot: args[0], + TargetSnapshot: args[1], + BlobsBefore: restic.NewBlobSet(), + BlobsAfter: restic.NewBlobSet(), + BlobsCommon: restic.NewBlobSet(), + } stats.BlobsBefore.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn1.Tree}) stats.BlobsAfter.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn2.Tree}) @@ -376,14 +410,21 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error { updateBlobs(repo, stats.BlobsBefore.Sub(both).Sub(stats.BlobsCommon), &stats.Removed) updateBlobs(repo, stats.BlobsAfter.Sub(both).Sub(stats.BlobsCommon), &stats.Added) - Printf("\n") - Printf("Files: %5d new, %5d removed, %5d changed\n", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles) - Printf("Dirs: %5d new, %5d removed\n", stats.Added.Dirs, stats.Removed.Dirs) - Printf("Others: %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others) - Printf("Data Blobs: %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs) - Printf("Tree Blobs: %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs) - Printf(" Added: %-5s\n", formatBytes(uint64(stats.Added.Bytes))) - Printf(" Removed: %-5s\n", formatBytes(uint64(stats.Removed.Bytes))) + if gopts.JSON { + err := json.NewEncoder(gopts.stdout).Encode(stats) + if err != nil { + Warnf("JSON encode failed: %v\n", err) + } + } else { + Printf("\n") + Printf("Files: %5d new, %5d removed, %5d changed\n", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles) + Printf("Dirs: %5d new, %5d removed\n", stats.Added.Dirs, stats.Removed.Dirs) + Printf("Others: %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others) + Printf("Data Blobs: %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs) + Printf("Tree Blobs: %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs) + Printf(" Added: %-5s\n", formatBytes(uint64(stats.Added.Bytes))) + Printf(" Removed: %-5s\n", formatBytes(uint64(stats.Removed.Bytes))) + } return nil } From 5f34ad523f29df07645b965485ed366dbd3215d1 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 5 Feb 2022 23:02:07 +0100 Subject: [PATCH 2/3] diff: fix test failure and add remark on quiet to changelog --- changelog/unreleased/issue-2508 | 7 +++++-- cmd/restic/integration_test.go | 13 ++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/changelog/unreleased/issue-2508 b/changelog/unreleased/issue-2508 index 5adaddd9a..d0292d002 100644 --- a/changelog/unreleased/issue-2508 +++ b/changelog/unreleased/issue-2508 @@ -1,8 +1,11 @@ -Enhancement: Support JSON output for diff +Enhancement: Support JSON output and quiet mode for diff We've added support for getting machine-readable output for snapshot diff, just pass the flag `--json` for `restic diff` and restic will output a JSON-encoded diff stats and change list. +Passing the `--quiet` flag to the `diff` command will only print the summary +and suppress the detailed output. + https://github.com/restic/restic/issues/2508 -https://github.com/restic/restic/pull/3592 \ No newline at end of file +https://github.com/restic/restic/pull/3592 diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 960fbc7e7..30aaa3225 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -2011,19 +2011,26 @@ func TestDiff(t *testing.T) { testRunBackup(t, "", []string{datadir}, opts, env.gopts) _, secondSnapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts)) + // quiet suppresses the diff output except for the summary + env.gopts.Quiet = false _, err := testRunDiffOutput(env.gopts, "", secondSnapshotID) rtest.Assert(t, err != nil, "expected error on invalid snapshot id") out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID) - if err != nil { - t.Fatalf("expected no error from diff for test repository, got %v", err) - } + rtest.OK(t, err) for _, pattern := range diffOutputRegexPatterns { r, err := regexp.Compile(pattern) rtest.Assert(t, err == nil, "failed to compile regexp %v", pattern) rtest.Assert(t, r.MatchString(out), "expected pattern %v in output, got\n%v", pattern, out) } + + // check quiet output + env.gopts.Quiet = true + outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID) + rtest.OK(t, err) + + rtest.Assert(t, len(outQuiet) < len(out), "expected shorter output on quiet mode %v vs. %v", len(outQuiet), len(out)) } type writeToOnly struct { From 74f29ad09bfba2c10e1de201b8c31e93ba781dfc Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 5 Feb 2022 23:26:31 +0100 Subject: [PATCH 3/3] diff: add basic test for json output format --- cmd/restic/integration_test.go | 63 ++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/cmd/restic/integration_test.go b/cmd/restic/integration_test.go index 30aaa3225..42936d2ea 100644 --- a/cmd/restic/integration_test.go +++ b/cmd/restic/integration_test.go @@ -159,8 +159,11 @@ func testRunDiffOutput(gopts GlobalOptions, firstSnapshotID string, secondSnapsh buf := bytes.NewBuffer(nil) globalOptions.stdout = buf + oldStdout := gopts.stdout + gopts.stdout = buf defer func() { globalOptions.stdout = os.Stdout + gopts.stdout = oldStdout }() opts := DiffOptions{ @@ -1972,10 +1975,8 @@ var diffOutputRegexPatterns = []string{ "Removed: +2[0-9]{2}\\.[0-9]{3} KiB", } -func TestDiff(t *testing.T) { +func setupDiffRepo(t *testing.T) (*testEnvironment, func(), string, string) { env, cleanup := withTestEnvironment(t) - defer cleanup() - testRunInit(t, env.gopts) datadir := filepath.Join(env.base, "testdata") @@ -2011,6 +2012,13 @@ func TestDiff(t *testing.T) { testRunBackup(t, "", []string{datadir}, opts, env.gopts) _, secondSnapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts)) + return env, cleanup, firstSnapshotID, secondSnapshotID +} + +func TestDiff(t *testing.T) { + env, cleanup, firstSnapshotID, secondSnapshotID := setupDiffRepo(t) + defer cleanup() + // quiet suppresses the diff output except for the summary env.gopts.Quiet = false _, err := testRunDiffOutput(env.gopts, "", secondSnapshotID) @@ -2033,6 +2041,55 @@ func TestDiff(t *testing.T) { rtest.Assert(t, len(outQuiet) < len(out), "expected shorter output on quiet mode %v vs. %v", len(outQuiet), len(out)) } +type typeSniffer struct { + MessageType string `json:"message_type"` +} + +func TestDiffJSON(t *testing.T) { + env, cleanup, firstSnapshotID, secondSnapshotID := setupDiffRepo(t) + defer cleanup() + + // quiet suppresses the diff output except for the summary + env.gopts.Quiet = false + env.gopts.JSON = true + out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID) + rtest.OK(t, err) + + var stat DiffStatsContainer + var changes int + + scanner := bufio.NewScanner(strings.NewReader(out)) + for scanner.Scan() { + line := scanner.Text() + var sniffer typeSniffer + rtest.OK(t, json.Unmarshal([]byte(line), &sniffer)) + switch sniffer.MessageType { + case "change": + changes++ + case "statistics": + rtest.OK(t, json.Unmarshal([]byte(line), &stat)) + default: + t.Fatalf("unexpected message type %v", sniffer.MessageType) + } + } + rtest.Equals(t, 9, changes) + rtest.Assert(t, stat.Added.Files == 2 && stat.Added.Dirs == 3 && stat.Added.DataBlobs == 2 && + stat.Removed.Files == 1 && stat.Removed.Dirs == 2 && stat.Removed.DataBlobs == 1 && + stat.ChangedFiles == 1, "unexpected statistics") + + // check quiet output + env.gopts.Quiet = true + outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID) + rtest.OK(t, err) + + stat = DiffStatsContainer{} + rtest.OK(t, json.Unmarshal([]byte(outQuiet), &stat)) + rtest.Assert(t, stat.Added.Files == 2 && stat.Added.Dirs == 3 && stat.Added.DataBlobs == 2 && + stat.Removed.Files == 1 && stat.Removed.Dirs == 2 && stat.Removed.DataBlobs == 1 && + stat.ChangedFiles == 1, "unexpected statistics") + rtest.Assert(t, stat.SourceSnapshot == firstSnapshotID && stat.TargetSnapshot == secondSnapshotID, "unexpected snapshot ids") +} + type writeToOnly struct { rd io.Reader }