Merge pull request 1962 from mholt/lsjson

ls: Stream output when using --json option

changelog/0.9.3_2018-10-13/pull-1962

@@ -0,0 +1,13 @@
+Enhancement: Stream JSON output for ls command
+
+The `ls` command now supports JSON output with the global `--json`
+flag, and this change streams out JSON messages one object at a time
+rather than en entire array buffered in memory before encoding. The
+advantage is it allows large listings to be handled efficiently.
+
+Two message types are printed: snapshots and nodes. A snapshot
+object will precede node objects which belong to that snapshot.
+The `struct_type` field can be used to determine which kind of
+message an object is.
+
+https://github.com/restic/restic/pull/1962

cmd/restic/cmd_ls.go

@@ -64,10 +64,8 @@ func init() {
 
 type lsSnapshot struct {
 	*restic.Snapshot
-
 	ID         *restic.ID `json:"id"`
 	ShortID    string     `json:"short_id"`
-	Nodes      []lsNode   `json:"nodes"`
 	StructType string     `json:"struct_type"` // "snapshot"
 }
 
@@ -150,24 +148,22 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 	var (
 		printSnapshot func(sn *restic.Snapshot)
 		printNode     func(path string, node *restic.Node)
-		printFinish   func() error
 	)
 
 	if gopts.JSON {
-		var lssnapshots []lsSnapshot
+		enc := json.NewEncoder(gopts.stdout)
 
 		printSnapshot = func(sn *restic.Snapshot) {
-			lss := lsSnapshot{
+			enc.Encode(lsSnapshot{
 				Snapshot:   sn,
 				ID:         sn.ID(),
 				ShortID:    sn.ID().Str(),
 				StructType: "snapshot",
-			}
+			})
-			lssnapshots = append(lssnapshots, lss)
 		}
 
 		printNode = func(path string, node *restic.Node) {
-			lsn := lsNode{
+			enc.Encode(lsNode{
 				Name:       node.Name,
 				Type:       node.Type,
 				Path:       path,
@@ -179,25 +175,15 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 				AccessTime: node.AccessTime,
 				ChangeTime: node.ChangeTime,
 				StructType: "node",
-			}
+			})
-			s := &lssnapshots[len(lssnapshots)-1]
-			s.Nodes = append(s.Nodes, lsn)
-		}
-
-		printFinish = func() error {
-			return json.NewEncoder(gopts.stdout).Encode(lssnapshots)
 		}
 	} else {
-		// default output methods
 		printSnapshot = func(sn *restic.Snapshot) {
 			Verbosef("snapshot %s of %v filtered by %v at %s):\n", sn.ID().Str(), sn.Paths, dirs, sn.Time)
 		}
 		printNode = func(path string, node *restic.Node) {
 			Printf("%s\n", formatNode(path, node, lsOptions.ListLong))
 		}
-		printFinish = func() error {
-			return nil
-		}
 	}
 
 	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args[:1]) {
@@ -240,5 +226,6 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
 			return err
 		}
 	}
-	return printFinish()
+
+	return nil
 }

cmd/restic/global.go

@@ -363,7 +363,9 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
 		if len(id) > 8 {
 			id = id[:8]
 		}
-		Verbosef("repository %v opened successfully, password is correct\n", id)
+		if !opts.JSON {
+			Verbosef("repository %v opened successfully, password is correct\n", id)
+		}
 	}
 
 	if opts.NoCache {