Remove Each(), add basic stats

backend/generic.go

@@ -24,30 +24,6 @@ var (
 
 const hashSize = sha256.Size
 
-// Each lists all entries of type t in the backend and calls function f() with
-// the id and data.
-func Each(be interface {
-	Lister
-	Getter
-}, t Type, f func(id ID, data []byte, err error)) error {
-	ids, err := be.List(t)
-	if err != nil {
-		return err
-	}
-
-	for _, id := range ids {
-		data, err := be.Get(t, id)
-		if err != nil {
-			f(id, nil, err)
-			continue
-		}
-
-		f(id, data, nil)
-	}
-
-	return nil
-}
-
 // Each lists all entries of type t in the backend and calls function f() with
 // the id.
 func EachID(be Lister, t Type, f func(ID)) error {

cmd/restic/cmd_key.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
@@ -32,10 +31,10 @@ func list_keys(s restic.Server) error {
 		return err
 	}
 
-	s.Each(backend.Key, func(id backend.ID, data []byte, err error) {
+	s.EachID(backend.Key, func(id backend.ID) {
-		k := restic.Key{}
+		k, err := restic.LoadKey(s, id)
-		err = json.Unmarshal(data, &k)
 		if err != nil {
+			fmt.Fprintf(os.Stderr, "LoadKey() failed: %v\n", err)
 			return
 		}
 

cmd/restic/cmd_list.go

@@ -33,17 +33,12 @@ func (cmd CmdList) Execute(args []string) error {
 		return err
 	}
 
-	var (
+	var t backend.Type
-		t    backend.Type
-		each func(backend.Type, func(backend.ID, []byte, error)) error = s.Each
-	)
 	switch args[0] {
 	case "data":
 		t = backend.Data
-		each = s.EachDecrypted
 	case "trees":
 		t = backend.Tree
-		each = s.EachDecrypted
 	case "snapshots":
 		t = backend.Snapshot
 	case "keys":
@@ -54,11 +49,7 @@ func (cmd CmdList) Execute(args []string) error {
 		return errors.New("invalid type")
 	}
 
-	return each(t, func(id backend.ID, data []byte, err error) {
+	return s.EachID(t, func(id backend.ID) {
-		if t == backend.Data || t == backend.Tree {
+		fmt.Printf("%s\n", id)
-			fmt.Printf("%s %s\n", id, backend.Hash(data))
-		} else {
-			fmt.Printf("%s\n", id)
-		}
 	})
 }

key.go

@@ -84,15 +84,7 @@ func CreateKey(s Server, password string) (*Key, error) {
 
 // OpenKey tries do decrypt the key specified by id with the given password.
 func OpenKey(s Server, id backend.ID, password string) (*Key, error) {
-	// extract data from repo
+	k, err := LoadKey(s, id)
-	data, err := s.Get(backend.Key, id)
-	if err != nil {
-		return nil, err
-	}
-
-	// restore json
-	k := &Key{}
-	err = json.Unmarshal(data, k)
 	if err != nil {
 		return nil, err
 	}
@@ -148,6 +140,24 @@ func SearchKey(s Server, password string) (*Key, error) {
 	return nil, ErrNoKeyFound
 }
 
+// LoadKey loads a key from the backend.
+func LoadKey(s Server, id backend.ID) (*Key, error) {
+	// extract data from repo
+	data, err := s.Get(backend.Key, id)
+	if err != nil {
+		return nil, err
+	}
+
+	// restore json
+	k := &Key{}
+	err = json.Unmarshal(data, k)
+	if err != nil {
+		return nil, err
+	}
+
+	return k, err
+}
+
 // AddKey adds a new key to an already existing repository.
 func AddKey(s Server, password string, template *Key) (*Key, error) {
 	// fill meta data about key

server.go

@@ -26,12 +26,6 @@ func NewServerWithKey(be backend.Backend, key *Key) Server {
 	return Server{be: be, key: key}
 }
 
-// Each lists all entries of type t in the backend and calls function f() with
-// the id and data.
-func (s Server) Each(t backend.Type, f func(id backend.ID, data []byte, err error)) error {
-	return backend.Each(s.be, t, f)
-}
-
 // Each lists all entries of type t in the backend and calls function f() with
 // the id.
 func (s Server) EachID(t backend.Type, f func(backend.ID)) error {
@@ -348,27 +342,53 @@ func (s Server) Key() *Key {
 	return s.key
 }
 
-// Each calls Each() with the given parameters, Decrypt() on the ciphertext
+type ServerStats struct {
-// and, on successful decryption, f with the plaintext.
+	Blobs, Trees uint
-func (s Server) EachDecrypted(t backend.Type, f func(backend.ID, []byte, error)) error {
+	Bytes        uint64
-	if s.key == nil {
+}
-		return errors.New("key for server not set")
+
+// Stats returns statistics for this backend and the server.
+func (s Server) Stats() (ServerStats, error) {
+	blobs := backend.NewIDSet()
+
+	// load all trees, in parallel
+	worker := func(wg *sync.WaitGroup, c <-chan backend.ID) {
+		for id := range c {
+			tree, err := LoadTree(s, id)
+			// ignore error and advance to next tree
+			if err != nil {
+				return
+			}
+
+			for _, id := range tree.Map.StorageIDs() {
+				blobs.Insert(id)
+			}
+		}
+		wg.Done()
 	}
 
-	return s.Each(t, func(id backend.ID, data []byte, e error) {
+	idCh := make(chan backend.ID)
-		if e != nil {
-			f(id, nil, e)
-			return
-		}
 
-		buf, err := s.key.Decrypt([]byte{}, data)
+	// start workers
-		if err != nil {
+	var wg sync.WaitGroup
-			f(id, nil, err)
+	for i := 0; i < maxConcurrency; i++ {
-			return
+		wg.Add(1)
-		}
+		go worker(&wg, idCh)
+	}
 
-		f(id, buf, nil)
+	// list ids
+	trees := 0
+	err := s.EachID(backend.Tree, func(id backend.ID) {
+		trees++
+		idCh <- id
 	})
+
+	close(idCh)
+
+	// wait for workers
+	wg.Wait()
+
+	return ServerStats{Blobs: uint(blobs.Len()), Trees: uint(trees)}, err
 }
 
 // Proxy methods to backend

server_test.go

@@ -123,3 +123,18 @@ func BenchmarkSaveFrom(t *testing.B) {
 		ok(t, err)
 	}
 }
+
+func TestServerStats(t *testing.T) {
+	be := setupBackend(t)
+	defer teardownBackend(t, be)
+	key := setupKey(t, be, "geheim")
+	server := restic.NewServerWithKey(be, key)
+
+	// archive a few files
+	sn := snapshot(t, server, *benchArchiveDirectory)
+	t.Logf("archived snapshot %v", sn.ID)
+
+	stats, err := server.Stats()
+	ok(t, err)
+	t.Logf("stats: %v", stats)
+}