mirror of
https://github.com/restic/restic.git
synced 2024-08-14 16:33:22 +02:00
208 lines
5.0 KiB
Go
208 lines
5.0 KiB
Go
package repository
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"io"
|
|
"os"
|
|
"runtime"
|
|
"sync"
|
|
|
|
"github.com/restic/restic/internal/backend"
|
|
"github.com/restic/restic/internal/errors"
|
|
"github.com/restic/restic/internal/hashing"
|
|
"github.com/restic/restic/internal/restic"
|
|
|
|
"github.com/restic/restic/internal/crypto"
|
|
"github.com/restic/restic/internal/debug"
|
|
"github.com/restic/restic/internal/fs"
|
|
"github.com/restic/restic/internal/pack"
|
|
|
|
"github.com/minio/sha256-simd"
|
|
)
|
|
|
|
// Packer holds a pack.Packer together with a hash writer.
|
|
type Packer struct {
|
|
*pack.Packer
|
|
tmpfile *os.File
|
|
bufWr *bufio.Writer
|
|
}
|
|
|
|
// packerManager keeps a list of open packs and creates new on demand.
|
|
type packerManager struct {
|
|
tpe restic.BlobType
|
|
key *crypto.Key
|
|
queueFn func(ctx context.Context, t restic.BlobType, p *Packer) error
|
|
|
|
pm sync.Mutex
|
|
packer *Packer
|
|
packSize uint
|
|
}
|
|
|
|
// newPackerManager returns a new packer manager which writes temporary files
|
|
// to a temporary directory
|
|
func newPackerManager(key *crypto.Key, tpe restic.BlobType, packSize uint, queueFn func(ctx context.Context, t restic.BlobType, p *Packer) error) *packerManager {
|
|
return &packerManager{
|
|
tpe: tpe,
|
|
key: key,
|
|
queueFn: queueFn,
|
|
packSize: packSize,
|
|
}
|
|
}
|
|
|
|
func (r *packerManager) Flush(ctx context.Context) error {
|
|
r.pm.Lock()
|
|
defer r.pm.Unlock()
|
|
|
|
if r.packer != nil {
|
|
debug.Log("manually flushing pending pack")
|
|
err := r.queueFn(ctx, r.tpe, r.packer)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
r.packer = nil
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (r *packerManager) SaveBlob(ctx context.Context, t restic.BlobType, id restic.ID, ciphertext []byte, uncompressedLength int) (int, error) {
|
|
r.pm.Lock()
|
|
defer r.pm.Unlock()
|
|
|
|
var err error
|
|
packer := r.packer
|
|
// use separate packer if compressed length is larger than the packsize
|
|
// this speeds up the garbage collection of oversized blobs and reduces the cache size
|
|
// as the oversize blobs are only downloaded if necessary
|
|
if len(ciphertext) >= int(r.packSize) || r.packer == nil {
|
|
packer, err = r.newPacker()
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
// don't store packer for oversized blob
|
|
if r.packer == nil {
|
|
r.packer = packer
|
|
}
|
|
}
|
|
|
|
// save ciphertext
|
|
// Add only appends bytes in memory to avoid being a scaling bottleneck
|
|
size, err := packer.Add(t, id, ciphertext, uncompressedLength)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
// if the pack and header is not full enough, put back to the list
|
|
if packer.Size() < r.packSize && !packer.HeaderFull() {
|
|
debug.Log("pack is not full enough (%d bytes)", packer.Size())
|
|
return size, nil
|
|
}
|
|
if packer == r.packer {
|
|
// forget full packer
|
|
r.packer = nil
|
|
}
|
|
|
|
// call while holding lock to prevent findPacker from creating new packers if the uploaders are busy
|
|
// else write the pack to the backend
|
|
err = r.queueFn(ctx, t, packer)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
return size + packer.HeaderOverhead(), nil
|
|
}
|
|
|
|
// findPacker returns a packer for a new blob of size bytes. Either a new one is
|
|
// created or one is returned that already has some blobs.
|
|
func (r *packerManager) newPacker() (packer *Packer, err error) {
|
|
debug.Log("create new pack")
|
|
tmpfile, err := fs.TempFile("", "restic-temp-pack-")
|
|
if err != nil {
|
|
return nil, errors.WithStack(err)
|
|
}
|
|
|
|
bufWr := bufio.NewWriter(tmpfile)
|
|
p := pack.NewPacker(r.key, bufWr)
|
|
packer = &Packer{
|
|
Packer: p,
|
|
tmpfile: tmpfile,
|
|
bufWr: bufWr,
|
|
}
|
|
|
|
return packer, nil
|
|
}
|
|
|
|
// savePacker stores p in the backend.
|
|
func (r *Repository) savePacker(ctx context.Context, t restic.BlobType, p *Packer) error {
|
|
debug.Log("save packer for %v with %d blobs (%d bytes)\n", t, p.Packer.Count(), p.Packer.Size())
|
|
err := p.Packer.Finalize()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
err = p.bufWr.Flush()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// calculate sha256 hash in a second pass
|
|
var rd io.Reader
|
|
rd, err = backend.NewFileReader(p.tmpfile, nil)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
beHasher := r.be.Hasher()
|
|
var beHr *hashing.Reader
|
|
if beHasher != nil {
|
|
beHr = hashing.NewReader(rd, beHasher)
|
|
rd = beHr
|
|
}
|
|
|
|
hr := hashing.NewReader(rd, sha256.New())
|
|
_, err = io.Copy(io.Discard, hr)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
id := restic.IDFromHash(hr.Sum(nil))
|
|
h := backend.Handle{Type: backend.PackFile, Name: id.String(), IsMetadata: t.IsMetadata()}
|
|
var beHash []byte
|
|
if beHr != nil {
|
|
beHash = beHr.Sum(nil)
|
|
}
|
|
rrd, err := backend.NewFileReader(p.tmpfile, beHash)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = r.be.Save(ctx, h, rrd)
|
|
if err != nil {
|
|
debug.Log("Save(%v) error: %v", h, err)
|
|
return err
|
|
}
|
|
|
|
debug.Log("saved as %v", h)
|
|
|
|
err = p.tmpfile.Close()
|
|
if err != nil {
|
|
return errors.Wrap(err, "close tempfile")
|
|
}
|
|
|
|
// on windows the tempfile is automatically deleted on close
|
|
if runtime.GOOS != "windows" {
|
|
err = fs.RemoveIfExists(p.tmpfile.Name())
|
|
if err != nil {
|
|
return errors.WithStack(err)
|
|
}
|
|
}
|
|
|
|
// update blobs in the index
|
|
debug.Log(" updating blobs %v to pack %v", p.Packer.Blobs(), id)
|
|
r.idx.StorePack(id, p.Packer.Blobs())
|
|
|
|
// Save index if full
|
|
if r.noAutoIndexUpdate {
|
|
return nil
|
|
}
|
|
return r.idx.SaveFullIndex(ctx, r)
|
|
}
|