Merge pull request 1962 from mholt/lsjson

ls: Stream output when using --json option
This commit is contained in:
Alexander Neumann 2018-10-14 15:51:38 +02:00
commit 41a4d67d93
3 changed files with 23 additions and 21 deletions

View File

@ -0,0 +1,13 @@
Enhancement: Stream JSON output for ls command
The `ls` command now supports JSON output with the global `--json`
flag, and this change streams out JSON messages one object at a time
rather than en entire array buffered in memory before encoding. The
advantage is it allows large listings to be handled efficiently.
Two message types are printed: snapshots and nodes. A snapshot
object will precede node objects which belong to that snapshot.
The `struct_type` field can be used to determine which kind of
message an object is.
https://github.com/restic/restic/pull/1962

View File

@ -64,10 +64,8 @@ func init() {
type lsSnapshot struct { type lsSnapshot struct {
*restic.Snapshot *restic.Snapshot
ID *restic.ID `json:"id"` ID *restic.ID `json:"id"`
ShortID string `json:"short_id"` ShortID string `json:"short_id"`
Nodes []lsNode `json:"nodes"`
StructType string `json:"struct_type"` // "snapshot" StructType string `json:"struct_type"` // "snapshot"
} }
@ -150,24 +148,22 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
var ( var (
printSnapshot func(sn *restic.Snapshot) printSnapshot func(sn *restic.Snapshot)
printNode func(path string, node *restic.Node) printNode func(path string, node *restic.Node)
printFinish func() error
) )
if gopts.JSON { if gopts.JSON {
var lssnapshots []lsSnapshot enc := json.NewEncoder(gopts.stdout)
printSnapshot = func(sn *restic.Snapshot) { printSnapshot = func(sn *restic.Snapshot) {
lss := lsSnapshot{ enc.Encode(lsSnapshot{
Snapshot: sn, Snapshot: sn,
ID: sn.ID(), ID: sn.ID(),
ShortID: sn.ID().Str(), ShortID: sn.ID().Str(),
StructType: "snapshot", StructType: "snapshot",
} })
lssnapshots = append(lssnapshots, lss)
} }
printNode = func(path string, node *restic.Node) { printNode = func(path string, node *restic.Node) {
lsn := lsNode{ enc.Encode(lsNode{
Name: node.Name, Name: node.Name,
Type: node.Type, Type: node.Type,
Path: path, Path: path,
@ -179,25 +175,15 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
AccessTime: node.AccessTime, AccessTime: node.AccessTime,
ChangeTime: node.ChangeTime, ChangeTime: node.ChangeTime,
StructType: "node", StructType: "node",
} })
s := &lssnapshots[len(lssnapshots)-1]
s.Nodes = append(s.Nodes, lsn)
}
printFinish = func() error {
return json.NewEncoder(gopts.stdout).Encode(lssnapshots)
} }
} else { } else {
// default output methods
printSnapshot = func(sn *restic.Snapshot) { printSnapshot = func(sn *restic.Snapshot) {
Verbosef("snapshot %s of %v filtered by %v at %s):\n", sn.ID().Str(), sn.Paths, dirs, sn.Time) Verbosef("snapshot %s of %v filtered by %v at %s):\n", sn.ID().Str(), sn.Paths, dirs, sn.Time)
} }
printNode = func(path string, node *restic.Node) { printNode = func(path string, node *restic.Node) {
Printf("%s\n", formatNode(path, node, lsOptions.ListLong)) Printf("%s\n", formatNode(path, node, lsOptions.ListLong))
} }
printFinish = func() error {
return nil
}
} }
for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args[:1]) { for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args[:1]) {
@ -240,5 +226,6 @@ func runLs(opts LsOptions, gopts GlobalOptions, args []string) error {
return err return err
} }
} }
return printFinish()
return nil
} }

View File

@ -363,8 +363,10 @@ func OpenRepository(opts GlobalOptions) (*repository.Repository, error) {
if len(id) > 8 { if len(id) > 8 {
id = id[:8] id = id[:8]
} }
if !opts.JSON {
Verbosef("repository %v opened successfully, password is correct\n", id) Verbosef("repository %v opened successfully, password is correct\n", id)
} }
}
if opts.NoCache { if opts.NoCache {
return s, nil return s, nil