diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index d190877a4..05f58c976 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -162,9 +162,18 @@ func (cmd CmdCat) Execute(args []string) error { return err case "blob": - data, err := repo.LoadBlob(pack.Data, id) - if err == nil { - _, err = os.Stdout.Write(data) + _, tpe, _, length, err := repo.Index().Lookup(id) + if err != nil { + return err + } + + if tpe != pack.Data { + return errors.New("wrong type for blob") + } + + buf := make([]byte, length) + data, err := repo.LoadBlob(pack.Data, id, buf) + if err != nil { return err } diff --git a/cmd/restic/fuse/file.go b/cmd/restic/fuse/file.go index 61b2ba170..895c844aa 100644 --- a/cmd/restic/fuse/file.go +++ b/cmd/restic/fuse/file.go @@ -1,6 +1,8 @@ package fuse import ( + "sync" + "github.com/restic/restic" "github.com/restic/restic/pack" "github.com/restic/restic/repository" @@ -12,6 +14,7 @@ import ( // Statically ensure that *file implements the given interface var _ = fs.HandleReader(&file{}) +var _ = fs.HandleReleaser(&file{}) type file struct { repo *repository.Repository @@ -21,6 +24,14 @@ type file struct { blobs [][]byte } +const defaultBlobSize = 128 * 1024 + +var blobPool = sync.Pool{ + New: func() interface{} { + return make([]byte, defaultBlobSize) + }, +} + func newFile(repo *repository.Repository, node *restic.Node) (*file, error) { sizes := make([]uint32, len(node.Content)) for i, blobID := range node.Content { @@ -48,50 +59,69 @@ func (f *file) Attr(ctx context.Context, a *fuse.Attr) error { func (f *file) getBlobAt(i int) (blob []byte, err error) { if f.blobs[i] != nil { - blob = f.blobs[i] - } else { - blob, err = f.repo.LoadBlob(pack.Data, f.node.Content[i]) - if err != nil { - return nil, err - } - f.blobs[i] = blob + return f.blobs[i], nil } + buf := blobPool.Get().([]byte) + buf = buf[:cap(buf)] + + if uint32(len(buf)) < f.sizes[i] { + if len(buf) > defaultBlobSize { + blobPool.Put(buf) + } + buf = make([]byte, f.sizes[i]) + } + + blob, err = f.repo.LoadBlob(pack.Data, f.node.Content[i], buf) + if err != nil { + return nil, err + } + f.blobs[i] = blob + return blob, nil } func (f *file) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { - off := req.Offset + offset := req.Offset // Skip blobs before the offset startContent := 0 - for off > int64(f.sizes[startContent]) { - off -= int64(f.sizes[startContent]) + for offset > int64(f.sizes[startContent]) { + offset -= int64(f.sizes[startContent]) startContent++ } - content := make([]byte, req.Size) - allContent := content - for i := startContent; i < len(f.sizes); i++ { + dst := resp.Data[0:req.Size] + readBytes := 0 + remainingBytes := req.Size + for i := startContent; remainingBytes > 0 && i < len(f.sizes); i++ { blob, err := f.getBlobAt(i) if err != nil { return err } - blob = blob[off:] - off = 0 + if offset > 0 { + blob = blob[offset:len(blob)] + offset = 0 + } - var copied int - if len(blob) > len(content) { - copied = copy(content[0:], blob[:len(content)]) - } else { - copied = copy(content[0:], blob) - } - content = content[copied:] - if len(content) == 0 { - break - } + copied := copy(dst, blob) + remainingBytes -= copied + readBytes += copied + + dst = dst[copied:] + } + resp.Data = resp.Data[:readBytes] + + return nil +} + +func (f *file) Release(ctx context.Context, req *fuse.ReleaseRequest) error { + for i := range f.blobs { + if f.blobs[i] != nil { + blobPool.Put(f.blobs[i]) + f.blobs[i] = nil + } } - resp.Data = allContent return nil } diff --git a/node.go b/node.go index a89d52e9a..e607bc8b1 100644 --- a/node.go +++ b/node.go @@ -209,8 +209,19 @@ func (node Node) createFileAt(path string, repo *repository.Repository) error { return errors.Annotate(err, "OpenFile") } + var buf []byte for _, id := range node.Content { - buf, err := repo.LoadBlob(pack.Data, id) + _, _, _, length, err := repo.Index().Lookup(id) + if err != nil { + return err + } + + buf = buf[:cap(buf)] + if uint(len(buf)) < length { + buf = make([]byte, length) + } + + buf, err := repo.LoadBlob(pack.Data, id, buf) if err != nil { return errors.Annotate(err, "Load") } diff --git a/repository/repository.go b/repository/repository.go index a53e3d4b2..faef38f61 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -55,7 +55,6 @@ func (r *Repository) PrefixLength(t backend.Type) (int, error) { func (r *Repository) LoadAndDecrypt(t backend.Type, id backend.ID) ([]byte, error) { debug.Log("Repo.Load", "load %v with id %v", t, id.Str()) - // load blob from pack rd, err := r.be.Get(t, id.String()) if err != nil { debug.Log("Repo.Load", "error loading %v: %v", id.Str(), err) @@ -87,8 +86,9 @@ func (r *Repository) LoadAndDecrypt(t backend.Type, id backend.ID) ([]byte, erro } // LoadBlob tries to load and decrypt content identified by t and id from a -// pack from the backend. -func (r *Repository) LoadBlob(t pack.BlobType, id backend.ID) ([]byte, error) { +// pack from the backend, the result is stored in buf, which must be large +// enough to hold the complete blob. +func (r *Repository) LoadBlob(t pack.BlobType, id backend.ID, buf []byte) ([]byte, error) { debug.Log("Repo.LoadBlob", "load %v with id %v", t, id.Str()) // lookup pack packID, tpe, offset, length, err := r.idx.Lookup(id) @@ -97,6 +97,10 @@ func (r *Repository) LoadBlob(t pack.BlobType, id backend.ID) ([]byte, error) { return nil, err } + if length > uint(cap(buf))+crypto.Extension { + return nil, errors.New("buf is too small") + } + if tpe != t { debug.Log("Repo.LoadBlob", "wrong type returned for %v: wanted %v, got %v", id.Str(), t, tpe) return nil, fmt.Errorf("blob has wrong type %v (wanted: %v)", tpe, t) @@ -111,7 +115,9 @@ func (r *Repository) LoadBlob(t pack.BlobType, id backend.ID) ([]byte, error) { return nil, err } - buf, err := ioutil.ReadAll(rd) + // make buffer that is large enough for the complete blob + cbuf := make([]byte, length) + _, err = io.ReadFull(rd, cbuf) if err != nil { return nil, err } @@ -122,17 +128,17 @@ func (r *Repository) LoadBlob(t pack.BlobType, id backend.ID) ([]byte, error) { } // decrypt - plain, err := r.Decrypt(buf) + buf, err = r.decryptTo(buf, cbuf) if err != nil { return nil, err } // check hash - if !backend.Hash(plain).Equal(id) { + if !backend.Hash(buf).Equal(id) { return nil, errors.New("invalid data returned") } - return plain, nil + return buf, nil } // LoadJSONUnpacked decrypts the data and afterwards calls json.Unmarshal on @@ -580,6 +586,12 @@ func (r *Repository) Init(password string) error { // Decrypt authenticates and decrypts ciphertext and returns the plaintext. func (r *Repository) Decrypt(ciphertext []byte) ([]byte, error) { + return r.decryptTo(nil, ciphertext) +} + +// decrypt authenticates and decrypts ciphertext and stores the result in +// plaintext. +func (r *Repository) decryptTo(plaintext, ciphertext []byte) ([]byte, error) { if r.key == nil { return nil, errors.New("key for repository not set") } diff --git a/repository/repository_test.go b/repository/repository_test.go index d978f83e4..f1012c90d 100644 --- a/repository/repository_test.go +++ b/repository/repository_test.go @@ -90,7 +90,7 @@ func TestSave(t *testing.T) { OK(t, repo.Flush()) // read back - buf, err := repo.LoadBlob(pack.Data, id) + buf, err := repo.LoadBlob(pack.Data, id, make([]byte, size)) Assert(t, len(buf) == len(data), "number of bytes read back does not match: expected %d, got %d", @@ -120,7 +120,7 @@ func TestSaveFrom(t *testing.T) { OK(t, repo.Flush()) // read back - buf, err := repo.LoadBlob(pack.Data, id) + buf, err := repo.LoadBlob(pack.Data, id, make([]byte, size)) Assert(t, len(buf) == len(data), "number of bytes read back does not match: expected %d, got %d",