Merge pull request #3592 from vgt/jsondiff

Add json output for diff command

changelog/unreleased/issue-2508

@@ -0,0 +1,11 @@
+Enhancement: Support JSON output and quiet mode for diff
+
+We've added support for getting machine-readable output for snapshot diff, just pass the
+flag `--json` for `restic diff` and restic will output a JSON-encoded diff stats and change
+list.
+
+Passing the `--quiet` flag to the `diff` command will only print the summary
+and suppress the detailed output.
+
+https://github.com/restic/restic/issues/2508
+https://github.com/restic/restic/pull/3592

cmd/restic/cmd_diff.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"encoding/json"
 	"path"
 	"reflect"
 	"sort"
@@ -62,15 +63,29 @@ func loadSnapshot(ctx context.Context, repo *repository.Repository, desc string)
 
 // Comparer collects all things needed to compare two snapshots.
 type Comparer struct {
-	repo restic.Repository
+	repo        restic.Repository
-	opts DiffOptions
+	opts        DiffOptions
+	printChange func(change *Change)
+}
+
+type Change struct {
+	MessageType string `json:"message_type"` // "change"
+	Path        string `json:"path"`
+	Modifier    string `json:"modifier"`
+}
+
+func NewChange(path string, mode string) *Change {
+	return &Change{MessageType: "change", Path: path, Modifier: mode}
 }
 
 // DiffStat collects stats for all types of items.
 type DiffStat struct {
-	Files, Dirs, Others  int
+	Files     int    `json:"files"`
-	DataBlobs, TreeBlobs int
+	Dirs      int    `json:"dirs"`
-	Bytes                uint64
+	Others    int    `json:"others"`
+	DataBlobs int    `json:"data_blobs"`
+	TreeBlobs int    `json:"tree_blobs"`
+	Bytes     uint64 `json:"bytes"`
 }
 
 // Add adds stats information for node to s.
@@ -113,21 +128,14 @@ func addBlobs(bs restic.BlobSet, node *restic.Node) {
 	}
 }
 
-// DiffStats collects the differences between two snapshots.
+type DiffStatsContainer struct {
-type DiffStats struct {
+	MessageType                          string         `json:"message_type"` // "statistics"
-	ChangedFiles                         int
+	SourceSnapshot                       string         `json:"source_snapshot"`
-	Added                                DiffStat
+	TargetSnapshot                       string         `json:"target_snapshot"`
-	Removed                              DiffStat
+	ChangedFiles                         int            `json:"changed_files"`
-	BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet
+	Added                                DiffStat       `json:"added"`
-}
+	Removed                              DiffStat       `json:"removed"`
-
+	BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet `json:"-"`
-// NewDiffStats creates new stats for a diff run.
-func NewDiffStats() *DiffStats {
-	return &DiffStats{
-		BlobsBefore: restic.NewBlobSet(),
-		BlobsAfter:  restic.NewBlobSet(),
-		BlobsCommon: restic.NewBlobSet(),
-	}
 }
 
 // updateBlobs updates the blob counters in the stats struct.
@@ -162,7 +170,7 @@ func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, b
 		if node.Type == "dir" {
 			name += "/"
 		}
-		Printf("%-5s%v\n", mode, name)
+		c.printChange(NewChange(name, "+"))
 		stats.Add(node)
 		addBlobs(blobs, node)
 
@@ -221,7 +229,7 @@ func uniqueNodeNames(tree1, tree2 *restic.Tree) (tree1Nodes, tree2Nodes map[stri
 	return tree1Nodes, tree2Nodes, uniqueNames
 }
 
-func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string, id1, id2 restic.ID) error {
+func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, prefix string, id1, id2 restic.ID) error {
 	debug.Log("diffing %v to %v", id1, id2)
 	tree1, err := c.repo.LoadTree(ctx, id1)
 	if err != nil {
@@ -265,7 +273,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string
 			}
 
 			if mod != "" {
-				Printf("%-5s%v\n", mod, name)
+				c.printChange(NewChange(name, mod))
 			}
 
 			if node1.Type == "dir" && node2.Type == "dir" {
@@ -284,7 +292,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string
 			if node1.Type == "dir" {
 				prefix += "/"
 			}
-			Printf("%-5s%v\n", "-", prefix)
+			c.printChange(NewChange(prefix, "-"))
 			stats.Removed.Add(node1)
 
 			if node1.Type == "dir" {
@@ -298,7 +306,7 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string
 			if node2.Type == "dir" {
 				prefix += "/"
 			}
-			Printf("%-5s%v\n", "+", prefix)
+			c.printChange(NewChange(prefix, "+"))
 			stats.Added.Add(node2)
 
 			if node2.Type == "dir" {
@@ -348,7 +356,9 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
+	if !gopts.JSON {
+		Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())
+	}
 
 	if sn1.Tree == nil {
 		return errors.Errorf("snapshot %v has nil tree", sn1.ID().Str())
@@ -361,9 +371,33 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 	c := &Comparer{
 		repo: repo,
 		opts: diffOptions,
+		printChange: func(change *Change) {
+			Printf("%-5s%v\n", change.Modifier, change.Path)
+		},
 	}
 
-	stats := NewDiffStats()
+	if gopts.JSON {
+		enc := json.NewEncoder(gopts.stdout)
+		c.printChange = func(change *Change) {
+			err := enc.Encode(change)
+			if err != nil {
+				Warnf("JSON encode failed: %v\n", err)
+			}
+		}
+	}
+
+	if gopts.Quiet {
+		c.printChange = func(change *Change) {}
+	}
+
+	stats := &DiffStatsContainer{
+		MessageType:    "statistics",
+		SourceSnapshot: args[0],
+		TargetSnapshot: args[1],
+		BlobsBefore:    restic.NewBlobSet(),
+		BlobsAfter:     restic.NewBlobSet(),
+		BlobsCommon:    restic.NewBlobSet(),
+	}
 	stats.BlobsBefore.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn1.Tree})
 	stats.BlobsAfter.Insert(restic.BlobHandle{Type: restic.TreeBlob, ID: *sn2.Tree})
 
@@ -376,14 +410,21 @@ func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
 	updateBlobs(repo, stats.BlobsBefore.Sub(both).Sub(stats.BlobsCommon), &stats.Removed)
 	updateBlobs(repo, stats.BlobsAfter.Sub(both).Sub(stats.BlobsCommon), &stats.Added)
 
-	Printf("\n")
+	if gopts.JSON {
-	Printf("Files:       %5d new, %5d removed, %5d changed\n", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles)
+		err := json.NewEncoder(gopts.stdout).Encode(stats)
-	Printf("Dirs:        %5d new, %5d removed\n", stats.Added.Dirs, stats.Removed.Dirs)
+		if err != nil {
-	Printf("Others:      %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
+			Warnf("JSON encode failed: %v\n", err)
-	Printf("Data Blobs:  %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
+		}
-	Printf("Tree Blobs:  %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
+	} else {
-	Printf("  Added:   %-5s\n", formatBytes(uint64(stats.Added.Bytes)))
+		Printf("\n")
-	Printf("  Removed: %-5s\n", formatBytes(uint64(stats.Removed.Bytes)))
+		Printf("Files:       %5d new, %5d removed, %5d changed\n", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles)
+		Printf("Dirs:        %5d new, %5d removed\n", stats.Added.Dirs, stats.Removed.Dirs)
+		Printf("Others:      %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
+		Printf("Data Blobs:  %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
+		Printf("Tree Blobs:  %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
+		Printf("  Added:   %-5s\n", formatBytes(uint64(stats.Added.Bytes)))
+		Printf("  Removed: %-5s\n", formatBytes(uint64(stats.Removed.Bytes)))
+	}
 
 	return nil
 }

cmd/restic/integration_test.go

@@ -159,8 +159,11 @@ func testRunDiffOutput(gopts GlobalOptions, firstSnapshotID string, secondSnapsh
 	buf := bytes.NewBuffer(nil)
 
 	globalOptions.stdout = buf
+	oldStdout := gopts.stdout
+	gopts.stdout = buf
 	defer func() {
 		globalOptions.stdout = os.Stdout
+		gopts.stdout = oldStdout
 	}()
 
 	opts := DiffOptions{
@@ -1972,10 +1975,8 @@ var diffOutputRegexPatterns = []string{
 	"Removed: +2[0-9]{2}\\.[0-9]{3} KiB",
 }
 
-func TestDiff(t *testing.T) {
+func setupDiffRepo(t *testing.T) (*testEnvironment, func(), string, string) {
 	env, cleanup := withTestEnvironment(t)
-	defer cleanup()
-
 	testRunInit(t, env.gopts)
 
 	datadir := filepath.Join(env.base, "testdata")
@@ -2011,19 +2012,82 @@ func TestDiff(t *testing.T) {
 	testRunBackup(t, "", []string{datadir}, opts, env.gopts)
 	_, secondSnapshotID := lastSnapshot(snapshots, loadSnapshotMap(t, env.gopts))
 
+	return env, cleanup, firstSnapshotID, secondSnapshotID
+}
+
+func TestDiff(t *testing.T) {
+	env, cleanup, firstSnapshotID, secondSnapshotID := setupDiffRepo(t)
+	defer cleanup()
+
+	// quiet suppresses the diff output except for the summary
+	env.gopts.Quiet = false
 	_, err := testRunDiffOutput(env.gopts, "", secondSnapshotID)
 	rtest.Assert(t, err != nil, "expected error on invalid snapshot id")
 
 	out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
-	if err != nil {
+	rtest.OK(t, err)
-		t.Fatalf("expected no error from diff for test repository, got %v", err)
-	}
 
 	for _, pattern := range diffOutputRegexPatterns {
 		r, err := regexp.Compile(pattern)
 		rtest.Assert(t, err == nil, "failed to compile regexp %v", pattern)
 		rtest.Assert(t, r.MatchString(out), "expected pattern %v in output, got\n%v", pattern, out)
 	}
+
+	// check quiet output
+	env.gopts.Quiet = true
+	outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
+	rtest.OK(t, err)
+
+	rtest.Assert(t, len(outQuiet) < len(out), "expected shorter output on quiet mode %v vs. %v", len(outQuiet), len(out))
+}
+
+type typeSniffer struct {
+	MessageType string `json:"message_type"`
+}
+
+func TestDiffJSON(t *testing.T) {
+	env, cleanup, firstSnapshotID, secondSnapshotID := setupDiffRepo(t)
+	defer cleanup()
+
+	// quiet suppresses the diff output except for the summary
+	env.gopts.Quiet = false
+	env.gopts.JSON = true
+	out, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
+	rtest.OK(t, err)
+
+	var stat DiffStatsContainer
+	var changes int
+
+	scanner := bufio.NewScanner(strings.NewReader(out))
+	for scanner.Scan() {
+		line := scanner.Text()
+		var sniffer typeSniffer
+		rtest.OK(t, json.Unmarshal([]byte(line), &sniffer))
+		switch sniffer.MessageType {
+		case "change":
+			changes++
+		case "statistics":
+			rtest.OK(t, json.Unmarshal([]byte(line), &stat))
+		default:
+			t.Fatalf("unexpected message type %v", sniffer.MessageType)
+		}
+	}
+	rtest.Equals(t, 9, changes)
+	rtest.Assert(t, stat.Added.Files == 2 && stat.Added.Dirs == 3 && stat.Added.DataBlobs == 2 &&
+		stat.Removed.Files == 1 && stat.Removed.Dirs == 2 && stat.Removed.DataBlobs == 1 &&
+		stat.ChangedFiles == 1, "unexpected statistics")
+
+	// check quiet output
+	env.gopts.Quiet = true
+	outQuiet, err := testRunDiffOutput(env.gopts, firstSnapshotID, secondSnapshotID)
+	rtest.OK(t, err)
+
+	stat = DiffStatsContainer{}
+	rtest.OK(t, json.Unmarshal([]byte(outQuiet), &stat))
+	rtest.Assert(t, stat.Added.Files == 2 && stat.Added.Dirs == 3 && stat.Added.DataBlobs == 2 &&
+		stat.Removed.Files == 1 && stat.Removed.Dirs == 2 && stat.Removed.DataBlobs == 1 &&
+		stat.ChangedFiles == 1, "unexpected statistics")
+	rtest.Assert(t, stat.SourceSnapshot == firstSnapshotID && stat.TargetSnapshot == secondSnapshotID, "unexpected snapshot ids")
 }
 
 type writeToOnly struct {