Decrease allocation rate in internal/pack

internal/repository benchmark results:

name             old time/op    new time/op    delta
PackerManager-8     179ms ± 1%     181ms ± 1%   +0.78%  (p=0.009 n=10+10)

name             old speed      new speed      delta
PackerManager-8   294MB/s ± 1%   292MB/s ± 1%   -0.77%  (p=0.009 n=10+10)

name             old alloc/op   new alloc/op   delta
PackerManager-8    91.3kB ± 0%    72.2kB ± 0%  -20.92%  (p=0.000 n=9+7)

name             old allocs/op  new allocs/op  delta
PackerManager-8     1.38k ± 0%     0.76k ± 0%  -45.20%  (p=0.000 n=10+7)
This commit is contained in:
greatroar 2020-07-22 18:46:55 +02:00 committed by Alexander Neumann
parent 9a8a2cae4c
commit ab2b7d7f9a
3 changed files with 85 additions and 52 deletions

View File

@ -1,7 +1,6 @@
package pack
import (
"bytes"
"encoding/binary"
"fmt"
"io"
@ -49,7 +48,8 @@ func (p *Packer) Add(t restic.BlobType, id restic.ID, data []byte) (int, error)
var entrySize = uint(binary.Size(restic.BlobType(0)) + binary.Size(uint32(0)) + len(restic.ID{}))
// headerEntry is used with encoding/binary to read and write header entries
// headerEntry describes the format of header entries. It serves only as
// documentation.
type headerEntry struct {
Type uint8
Length uint32
@ -64,16 +64,15 @@ func (p *Packer) Finalize() (uint, error) {
bytesWritten := p.bytes
hdrBuf := bytes.NewBuffer(nil)
bytesHeader, err := p.writeHeader(hdrBuf)
header, err := p.makeHeader()
if err != nil {
return 0, err
}
encryptedHeader := make([]byte, 0, hdrBuf.Len()+p.k.Overhead()+p.k.NonceSize())
encryptedHeader := make([]byte, 0, len(header)+p.k.Overhead()+p.k.NonceSize())
nonce := crypto.NewRandomNonce()
encryptedHeader = append(encryptedHeader, nonce...)
encryptedHeader = p.k.Seal(encryptedHeader, nonce, hdrBuf.Bytes(), nil)
encryptedHeader = p.k.Seal(encryptedHeader, nonce, header, nil)
// append the header
n, err := p.wr.Write(encryptedHeader)
@ -81,7 +80,7 @@ func (p *Packer) Finalize() (uint, error) {
return 0, errors.Wrap(err, "Write")
}
hdrBytes := restic.CiphertextLength(int(bytesHeader))
hdrBytes := restic.CiphertextLength(len(header))
if n != hdrBytes {
return 0, errors.New("wrong number of bytes written")
}
@ -99,32 +98,27 @@ func (p *Packer) Finalize() (uint, error) {
return bytesWritten, nil
}
// writeHeader constructs and writes the header to wr.
func (p *Packer) writeHeader(wr io.Writer) (bytesWritten uint, err error) {
for _, b := range p.blobs {
entry := headerEntry{
Length: uint32(b.Length),
ID: b.ID,
}
// makeHeader constructs the header for p.
func (p *Packer) makeHeader() ([]byte, error) {
buf := make([]byte, 0, len(p.blobs)*int(entrySize))
for _, b := range p.blobs {
switch b.Type {
case restic.DataBlob:
entry.Type = 0
buf = append(buf, 0)
case restic.TreeBlob:
entry.Type = 1
buf = append(buf, 1)
default:
return 0, errors.Errorf("invalid blob type %v", b.Type)
return nil, errors.Errorf("invalid blob type %v", b.Type)
}
err := binary.Write(wr, binary.LittleEndian, entry)
if err != nil {
return bytesWritten, errors.Wrap(err, "binary.Write")
}
bytesWritten += entrySize
var lenLE [4]byte
binary.LittleEndian.PutUint32(lenLE[:], uint32(b.Length))
buf = append(buf, lenLE[:]...)
buf = append(buf, b.ID[:]...)
}
return
return buf, nil
}
// Size returns the number of bytes written so far.
@ -275,40 +269,19 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries []restic.Blob, err
return nil, err
}
hdrRd := bytes.NewReader(buf)
entries = make([]restic.Blob, 0, uint(len(buf))/entrySize)
pos := uint(0)
for {
e := headerEntry{}
err = binary.Read(hdrRd, binary.LittleEndian, &e)
if errors.Cause(err) == io.EOF {
break
}
for len(buf) > 0 {
entry, err := parseHeaderEntry(buf)
if err != nil {
return nil, errors.Wrap(err, "binary.Read")
}
entry := restic.Blob{
Length: uint(e.Length),
ID: e.ID,
Offset: pos,
}
switch e.Type {
case 0:
entry.Type = restic.DataBlob
case 1:
entry.Type = restic.TreeBlob
default:
return nil, errors.Errorf("invalid type %d", e.Type)
return nil, err
}
entry.Offset = pos
entries = append(entries, entry)
pos += uint(e.Length)
pos += entry.Length
buf = buf[entrySize:]
}
return entries, nil
@ -318,3 +291,25 @@ func List(k *crypto.Key, rd io.ReaderAt, size int64) (entries []restic.Blob, err
func PackedSizeOfBlob(blobLength uint) uint {
return blobLength + entrySize
}
func parseHeaderEntry(p []byte) (b restic.Blob, err error) {
if uint(len(p)) < entrySize {
err = errors.Errorf("parseHeaderEntry: buffer of size %d too short", len(p))
return b, err
}
p = p[:entrySize]
switch p[0] {
case 0:
b.Type = restic.DataBlob
case 1:
b.Type = restic.TreeBlob
default:
return b, errors.Errorf("invalid type %d", p[0])
}
b.Length = uint(binary.LittleEndian.Uint32(p[1:5]))
copy(b.ID[:], p[5:])
return b, nil
}

View File

@ -7,9 +7,44 @@ import (
"testing"
"github.com/restic/restic/internal/crypto"
"github.com/restic/restic/internal/restic"
rtest "github.com/restic/restic/internal/test"
)
func TestParseHeaderEntry(t *testing.T) {
h := headerEntry{
Type: 0, // Blob.
Length: 100,
}
for i := range h.ID {
h.ID[i] = byte(i)
}
buf := new(bytes.Buffer)
_ = binary.Write(buf, binary.LittleEndian, &h)
b, err := parseHeaderEntry(buf.Bytes())
rtest.OK(t, err)
rtest.Equals(t, restic.DataBlob, b.Type)
t.Logf("%v %v", h.ID, b.ID)
rtest.Assert(t, bytes.Equal(h.ID[:], b.ID[:]), "id mismatch")
rtest.Equals(t, uint(h.Length), b.Length)
h.Type = 0xae
buf.Reset()
_ = binary.Write(buf, binary.LittleEndian, &h)
b, err = parseHeaderEntry(buf.Bytes())
rtest.Assert(t, err != nil, "no error for invalid type")
h.Type = 0
buf.Reset()
_ = binary.Write(buf, binary.LittleEndian, &h)
b, err = parseHeaderEntry(buf.Bytes()[:entrySize-1])
rtest.Assert(t, err != nil, "no error for short input")
}
type countingReaderAt struct {
delegate io.ReaderAt
invocationCount int

View File

@ -57,7 +57,10 @@ func (r *packerManager) findPacker() (packer *Packer, err error) {
// search for a suitable packer
if len(r.packers) > 0 {
p := r.packers[0]
r.packers = r.packers[1:]
last := len(r.packers) - 1
r.packers[0] = r.packers[last]
r.packers[last] = nil // Allow GC of stale reference.
r.packers = r.packers[:last]
return p, nil
}