mirror of
https://github.com/restic/restic.git
synced 2024-09-07 11:59:25 +02:00
HMAC-SHA256 calls SHA256() twice which is very expensive. Therefore, this commit uses Poly1305-AES instead of HMAC-SHA256. benchcmp: benchmark old ns/op new ns/op delta BenchmarkChunkEncrypt 261033772 195114818 -25.25% BenchmarkChunkEncryptParallel 260973195 195787368 -24.98% BenchmarkArchiveDirectory 1050500651 1002615884 -4.56% BenchmarkPreload 23544286 24994508 +6.16% BenchmarkLoadTree 350065 427665 +22.17% BenchmarkEncryptWriter 87789753 31069126 -64.61% BenchmarkEncrypt 88283197 38259043 -56.66% BenchmarkDecryptReader 90478843 40714818 -55.00% BenchmarkEncryptDecryptReader 179917626 81231730 -54.85% BenchmarkDecrypt 87871591 37784207 -57.00% BenchmarkSaveJSON 52481 56861 +8.35% BenchmarkSaveFrom 75404085 51108596 -32.22% BenchmarkLoadJSONID 90545437 82696805 -8.67% benchmark old MB/s new MB/s speedup BenchmarkChunkEncrypt 40.17 53.74 1.34x BenchmarkChunkEncryptParallel 40.18 53.56 1.33x BenchmarkEncryptWriter 95.55 270.00 2.83x BenchmarkEncrypt 95.02 219.26 2.31x BenchmarkDecryptReader 92.71 206.03 2.22x BenchmarkEncryptDecryptReader 46.62 103.27 2.22x BenchmarkDecrypt 95.46 222.01 2.33x BenchmarkSaveFrom 55.62 82.07 1.48x benchmark old allocs new allocs delta BenchmarkChunkEncrypt 112 110 -1.79% BenchmarkChunkEncryptParallel 103 100 -2.91% BenchmarkArchiveDirectory 383704 392083 +2.18% BenchmarkPreload 21765 21874 +0.50% BenchmarkLoadTree 341 436 +27.86% BenchmarkEncryptWriter 20 17 -15.00% BenchmarkEncrypt 14 13 -7.14% BenchmarkDecryptReader 18 15 -16.67% BenchmarkEncryptDecryptReader 46 39 -15.22% BenchmarkDecrypt 16 12 -25.00% BenchmarkSaveJSON 81 86 +6.17% BenchmarkSaveFrom 117 121 +3.42% BenchmarkLoadJSONID 80525 80264 -0.32% benchmark old bytes new bytes delta BenchmarkChunkEncrypt 118956 64697 -45.61% BenchmarkChunkEncryptParallel 118972 64681 -45.63% BenchmarkArchiveDirectory 160236600 177498232 +10.77% BenchmarkPreload 2772488 3302992 +19.13% BenchmarkLoadTree 49102 46484 -5.33% BenchmarkEncryptWriter 28927 8388146 +28897.64% BenchmarkEncrypt 2473 1950 -21.15% BenchmarkDecryptReader 527827 2774 -99.47% BenchmarkEncryptDecryptReader 4100875 1528036 -62.74% BenchmarkDecrypt 2509 2154 -14.15% BenchmarkSaveJSON 4971 5892 +18.53% BenchmarkSaveFrom 40117 31742 -20.88% BenchmarkLoadJSONID 9444217 9442106 -0.02% This closes #102.
421 lines
8.9 KiB
Go
421 lines
8.9 KiB
Go
package restic
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"sync"
|
|
|
|
"github.com/restic/restic/backend"
|
|
"github.com/restic/restic/debug"
|
|
)
|
|
|
|
type Server struct {
|
|
be backend.Backend
|
|
key *Key
|
|
}
|
|
|
|
func NewServer(be backend.Backend) Server {
|
|
return Server{be: be}
|
|
}
|
|
|
|
func NewServerWithKey(be backend.Backend, key *Key) Server {
|
|
return Server{be: be, key: key}
|
|
}
|
|
|
|
// Each lists all entries of type t in the backend and calls function f() with
|
|
// the id.
|
|
func (s Server) EachID(t backend.Type, f func(backend.ID)) error {
|
|
return backend.EachID(s.be, t, f)
|
|
}
|
|
|
|
// Find loads the list of all blobs of type t and searches for IDs which start
|
|
// with prefix. If none is found, nil and ErrNoIDPrefixFound is returned. If
|
|
// more than one is found, nil and ErrMultipleIDMatches is returned.
|
|
func (s Server) Find(t backend.Type, prefix string) (backend.ID, error) {
|
|
return backend.Find(s.be, t, prefix)
|
|
}
|
|
|
|
// FindSnapshot takes a string and tries to find a snapshot whose ID matches
|
|
// the string as closely as possible.
|
|
func (s Server) FindSnapshot(id string) (backend.ID, error) {
|
|
return backend.FindSnapshot(s.be, id)
|
|
}
|
|
|
|
// PrefixLength returns the number of bytes required so that all prefixes of
|
|
// all IDs of type t are unique.
|
|
func (s Server) PrefixLength(t backend.Type) (int, error) {
|
|
return backend.PrefixLength(s.be, t)
|
|
}
|
|
|
|
// Load tries to load and decrypt content identified by t and blob from the backend.
|
|
func (s Server) Load(t backend.Type, blob Blob) ([]byte, error) {
|
|
// load data
|
|
buf, err := s.Get(t, blob.Storage)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// check length
|
|
if len(buf) != int(blob.StorageSize) {
|
|
return nil, errors.New("Invalid storage length")
|
|
}
|
|
|
|
// decrypt
|
|
buf, err = s.Decrypt(buf)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// check length
|
|
if len(buf) != int(blob.Size) {
|
|
return nil, errors.New("Invalid length")
|
|
}
|
|
|
|
// check SHA256 sum
|
|
id := backend.Hash(buf)
|
|
if !blob.ID.Equal(id) {
|
|
return nil, fmt.Errorf("load %v: expected plaintext hash %v, got %v", blob.Storage, blob.ID, id)
|
|
}
|
|
|
|
return buf, nil
|
|
}
|
|
|
|
// Load tries to load and decrypt content identified by t and id from the backend.
|
|
func (s Server) LoadID(t backend.Type, storageID backend.ID) ([]byte, error) {
|
|
// load data
|
|
buf, err := s.Get(t, storageID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// decrypt
|
|
buf, err = s.Decrypt(buf)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return buf, nil
|
|
}
|
|
|
|
// LoadJSON calls Load() to get content from the backend and afterwards calls
|
|
// json.Unmarshal on the item.
|
|
func (s Server) LoadJSON(t backend.Type, blob Blob, item interface{}) error {
|
|
return s.LoadJSONID(t, blob.Storage, item)
|
|
}
|
|
|
|
// LoadJSONID calls Load() to get content from the backend and afterwards calls
|
|
// json.Unmarshal on the item.
|
|
func (s Server) LoadJSONID(t backend.Type, storageID backend.ID, item interface{}) error {
|
|
// read
|
|
rd, err := s.GetReader(t, storageID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer rd.Close()
|
|
|
|
// decrypt
|
|
decryptRd, err := s.key.DecryptFrom(rd)
|
|
defer decryptRd.Close()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// decode
|
|
decoder := json.NewDecoder(decryptRd)
|
|
err = decoder.Decode(item)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Save encrypts data and stores it to the backend as type t.
|
|
func (s Server) Save(t backend.Type, data []byte, id backend.ID) (Blob, error) {
|
|
if id == nil {
|
|
// compute plaintext hash
|
|
id = backend.Hash(data)
|
|
}
|
|
|
|
// create a new blob
|
|
blob := Blob{
|
|
ID: id,
|
|
Size: uint64(len(data)),
|
|
}
|
|
|
|
var ciphertext []byte
|
|
|
|
// if the data is small enough, use a slice from the pool
|
|
if len(data) <= maxCiphertextSize-ivSize-macSize {
|
|
ciphertext = GetChunkBuf("ch.Save()")
|
|
defer FreeChunkBuf("ch.Save()", ciphertext)
|
|
} else {
|
|
l := len(data) + ivSize + macSize
|
|
|
|
debug.Log("Server.Save", "create large slice of %d bytes for ciphertext", l)
|
|
|
|
// use a new slice
|
|
ciphertext = make([]byte, l)
|
|
}
|
|
|
|
// encrypt blob
|
|
n, err := s.Encrypt(ciphertext, data)
|
|
if err != nil {
|
|
return Blob{}, err
|
|
}
|
|
|
|
ciphertext = ciphertext[:n]
|
|
|
|
// save blob
|
|
backendBlob, err := s.Create(t)
|
|
if err != nil {
|
|
return Blob{}, err
|
|
}
|
|
|
|
_, err = backendBlob.Write(ciphertext)
|
|
if err != nil {
|
|
return Blob{}, err
|
|
}
|
|
|
|
err = backendBlob.Close()
|
|
if err != nil {
|
|
return Blob{}, err
|
|
}
|
|
|
|
sid, err := backendBlob.ID()
|
|
if err != nil {
|
|
return Blob{}, err
|
|
}
|
|
|
|
blob.Storage = sid
|
|
blob.StorageSize = uint64(len(ciphertext))
|
|
|
|
return blob, nil
|
|
}
|
|
|
|
// SaveFrom encrypts data read from rd and stores it to the backend as type t.
|
|
func (s Server) SaveFrom(t backend.Type, id backend.ID, length uint, rd io.Reader) (Blob, error) {
|
|
if id == nil {
|
|
return Blob{}, errors.New("id is nil")
|
|
}
|
|
|
|
backendBlob, err := s.Create(t)
|
|
if err != nil {
|
|
return Blob{}, err
|
|
}
|
|
|
|
encWr := s.key.EncryptTo(backendBlob)
|
|
|
|
_, err = io.Copy(encWr, rd)
|
|
if err != nil {
|
|
return Blob{}, err
|
|
}
|
|
|
|
// finish encryption
|
|
err = encWr.Close()
|
|
if err != nil {
|
|
return Blob{}, fmt.Errorf("EncryptedWriter.Close(): %v", err)
|
|
}
|
|
|
|
// finish backend blob
|
|
err = backendBlob.Close()
|
|
if err != nil {
|
|
return Blob{}, fmt.Errorf("backend.Blob.Close(): %v", err)
|
|
}
|
|
|
|
storageID, err := backendBlob.ID()
|
|
if err != nil {
|
|
return Blob{}, fmt.Errorf("backend.Blob.ID(): %v", err)
|
|
}
|
|
|
|
return Blob{
|
|
ID: id,
|
|
Size: uint64(length),
|
|
Storage: storageID,
|
|
StorageSize: uint64(backendBlob.Size()),
|
|
}, nil
|
|
}
|
|
|
|
// SaveJSON serialises item as JSON and encrypts and saves it in the backend as
|
|
// type t.
|
|
func (s Server) SaveJSON(t backend.Type, item interface{}) (Blob, error) {
|
|
backendBlob, err := s.Create(t)
|
|
if err != nil {
|
|
return Blob{}, fmt.Errorf("Create: %v", err)
|
|
}
|
|
|
|
encWr := s.key.EncryptTo(backendBlob)
|
|
hw := backend.NewHashingWriter(encWr, sha256.New())
|
|
|
|
enc := json.NewEncoder(hw)
|
|
err = enc.Encode(item)
|
|
if err != nil {
|
|
return Blob{}, fmt.Errorf("json.NewEncoder: %v", err)
|
|
}
|
|
|
|
// finish encryption
|
|
err = encWr.Close()
|
|
if err != nil {
|
|
return Blob{}, fmt.Errorf("EncryptedWriter.Close(): %v", err)
|
|
}
|
|
|
|
// finish backend blob
|
|
err = backendBlob.Close()
|
|
if err != nil {
|
|
return Blob{}, fmt.Errorf("backend.Blob.Close(): %v", err)
|
|
}
|
|
|
|
id := hw.Sum(nil)
|
|
storageID, err := backendBlob.ID()
|
|
if err != nil {
|
|
return Blob{}, fmt.Errorf("backend.Blob.ID(): %v", err)
|
|
}
|
|
|
|
return Blob{
|
|
ID: id,
|
|
Size: uint64(hw.Size()),
|
|
Storage: storageID,
|
|
StorageSize: uint64(backendBlob.Size()),
|
|
}, nil
|
|
}
|
|
|
|
// Returns the backend used for this server.
|
|
func (s Server) Backend() backend.Backend {
|
|
return s.be
|
|
}
|
|
|
|
func (s *Server) SearchKey(password string) error {
|
|
key, err := SearchKey(*s, password)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
s.key = key
|
|
|
|
return nil
|
|
}
|
|
|
|
func (s Server) Decrypt(ciphertext []byte) ([]byte, error) {
|
|
if s.key == nil {
|
|
return nil, errors.New("key for server not set")
|
|
}
|
|
|
|
return s.key.Decrypt([]byte{}, ciphertext)
|
|
}
|
|
|
|
func (s Server) Encrypt(ciphertext, plaintext []byte) (int, error) {
|
|
if s.key == nil {
|
|
return 0, errors.New("key for server not set")
|
|
}
|
|
|
|
return s.key.Encrypt(ciphertext, plaintext)
|
|
}
|
|
|
|
func (s Server) Key() *Key {
|
|
return s.key
|
|
}
|
|
|
|
type ServerStats struct {
|
|
Blobs, Trees uint
|
|
Bytes uint64
|
|
}
|
|
|
|
// Stats returns statistics for this backend and the server.
|
|
func (s Server) Stats() (ServerStats, error) {
|
|
blobs := backend.NewIDSet()
|
|
|
|
// load all trees, in parallel
|
|
worker := func(wg *sync.WaitGroup, c <-chan backend.ID) {
|
|
for id := range c {
|
|
tree, err := LoadTree(s, id)
|
|
// ignore error and advance to next tree
|
|
if err != nil {
|
|
return
|
|
}
|
|
|
|
for _, id := range tree.Map.StorageIDs() {
|
|
blobs.Insert(id)
|
|
}
|
|
}
|
|
wg.Done()
|
|
}
|
|
|
|
idCh := make(chan backend.ID)
|
|
|
|
// start workers
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < maxConcurrency; i++ {
|
|
wg.Add(1)
|
|
go worker(&wg, idCh)
|
|
}
|
|
|
|
// list ids
|
|
trees := 0
|
|
err := s.EachID(backend.Tree, func(id backend.ID) {
|
|
trees++
|
|
idCh <- id
|
|
})
|
|
|
|
close(idCh)
|
|
|
|
// wait for workers
|
|
wg.Wait()
|
|
|
|
return ServerStats{Blobs: uint(blobs.Len()), Trees: uint(trees)}, err
|
|
}
|
|
|
|
// Count counts the number of objects of type t in the backend.
|
|
func (s Server) Count(t backend.Type) (int, error) {
|
|
l, err := s.be.List(t)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
return len(l), nil
|
|
}
|
|
|
|
// Proxy methods to backend
|
|
|
|
func (s Server) List(t backend.Type) (backend.IDs, error) {
|
|
return s.be.List(t)
|
|
}
|
|
|
|
func (s Server) Get(t backend.Type, id backend.ID) ([]byte, error) {
|
|
return s.be.Get(t, id)
|
|
}
|
|
|
|
func (s Server) GetReader(t backend.Type, id backend.ID) (io.ReadCloser, error) {
|
|
return s.be.GetReader(t, id)
|
|
}
|
|
|
|
func (s Server) Create(t backend.Type) (backend.Blob, error) {
|
|
return s.be.Create(t)
|
|
}
|
|
|
|
func (s Server) Test(t backend.Type, id backend.ID) (bool, error) {
|
|
return s.be.Test(t, id)
|
|
}
|
|
|
|
func (s Server) Remove(t backend.Type, id backend.ID) error {
|
|
return s.be.Remove(t, id)
|
|
}
|
|
|
|
func (s Server) Close() error {
|
|
return s.be.Close()
|
|
}
|
|
|
|
func (s Server) Delete() error {
|
|
if b, ok := s.be.(backend.Deleter); ok {
|
|
return b.Delete()
|
|
}
|
|
|
|
return errors.New("Delete() called for backend that does not implement this method")
|
|
}
|
|
|
|
func (s Server) ID() backend.ID {
|
|
return s.be.ID()
|
|
}
|