mirror of
https://github.com/restic/restic.git
synced 2024-09-08 20:39:25 +02:00
A file is always cached whole. Thus, any out of bounds access will also fail when directed at the backend. To handle case in which the cached file is broken, then caller must call Cache.Forget(h) for the file in question.
289 lines
6.5 KiB
Go
289 lines
6.5 KiB
Go
package cache
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io"
|
|
"math/rand"
|
|
"os"
|
|
"runtime"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/restic/restic/internal/backend"
|
|
"github.com/restic/restic/internal/errors"
|
|
"github.com/restic/restic/internal/fs"
|
|
"github.com/restic/restic/internal/restic"
|
|
rtest "github.com/restic/restic/internal/test"
|
|
|
|
"golang.org/x/sync/errgroup"
|
|
)
|
|
|
|
func generateRandomFiles(t testing.TB, tpe backend.FileType, c *Cache) restic.IDSet {
|
|
ids := restic.NewIDSet()
|
|
for i := 0; i < rand.Intn(15)+10; i++ {
|
|
buf := rtest.Random(rand.Int(), 1<<19)
|
|
id := restic.Hash(buf)
|
|
h := backend.Handle{Type: tpe, Name: id.String()}
|
|
|
|
if c.Has(h) {
|
|
t.Errorf("index %v present before save", id)
|
|
}
|
|
|
|
err := c.save(h, bytes.NewReader(buf))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ids.Insert(id)
|
|
}
|
|
return ids
|
|
}
|
|
|
|
// randomID returns a random ID from s.
|
|
func randomID(s restic.IDSet) restic.ID {
|
|
for id := range s {
|
|
return id
|
|
}
|
|
panic("set is empty")
|
|
}
|
|
|
|
func load(t testing.TB, c *Cache, h backend.Handle) []byte {
|
|
rd, inCache, err := c.load(h, 0, 0)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rtest.Equals(t, true, inCache, "expected inCache flag to be true")
|
|
|
|
if rd == nil {
|
|
t.Fatalf("load() returned nil reader")
|
|
}
|
|
|
|
buf, err := io.ReadAll(rd)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if err = rd.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
return buf
|
|
}
|
|
|
|
func listFiles(t testing.TB, c *Cache, tpe restic.FileType) restic.IDSet {
|
|
list, err := c.list(tpe)
|
|
if err != nil {
|
|
t.Errorf("listing failed: %v", err)
|
|
}
|
|
|
|
return list
|
|
}
|
|
|
|
func clearFiles(t testing.TB, c *Cache, tpe restic.FileType, valid restic.IDSet) {
|
|
if err := c.Clear(tpe, valid); err != nil {
|
|
t.Error(err)
|
|
}
|
|
}
|
|
|
|
func TestFiles(t *testing.T) {
|
|
seed := time.Now().Unix()
|
|
t.Logf("seed is %v", seed)
|
|
rand.Seed(seed)
|
|
|
|
c := TestNewCache(t)
|
|
|
|
var tests = []restic.FileType{
|
|
restic.SnapshotFile,
|
|
restic.PackFile,
|
|
restic.IndexFile,
|
|
}
|
|
|
|
for _, tpe := range tests {
|
|
t.Run(tpe.String(), func(t *testing.T) {
|
|
ids := generateRandomFiles(t, tpe, c)
|
|
id := randomID(ids)
|
|
|
|
h := backend.Handle{Type: tpe, Name: id.String()}
|
|
id2 := restic.Hash(load(t, c, h))
|
|
|
|
if !id.Equal(id2) {
|
|
t.Errorf("wrong data returned, want %v, got %v", id.Str(), id2.Str())
|
|
}
|
|
|
|
if !c.Has(h) {
|
|
t.Errorf("cache thinks index %v isn't present", id.Str())
|
|
}
|
|
|
|
list := listFiles(t, c, tpe)
|
|
if !ids.Equals(list) {
|
|
t.Errorf("wrong list of index IDs returned, want:\n %v\ngot:\n %v", ids, list)
|
|
}
|
|
|
|
clearFiles(t, c, tpe, restic.NewIDSet(id))
|
|
list2 := listFiles(t, c, tpe)
|
|
ids.Delete(id)
|
|
want := restic.NewIDSet(id)
|
|
if !list2.Equals(want) {
|
|
t.Errorf("ClearIndexes removed indexes, want:\n %v\ngot:\n %v", list2, want)
|
|
}
|
|
|
|
clearFiles(t, c, tpe, restic.NewIDSet())
|
|
want = restic.NewIDSet()
|
|
list3 := listFiles(t, c, tpe)
|
|
if !list3.Equals(want) {
|
|
t.Errorf("ClearIndexes returned a wrong list, want:\n %v\ngot:\n %v", want, list3)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestFileLoad(t *testing.T) {
|
|
seed := time.Now().Unix()
|
|
t.Logf("seed is %v", seed)
|
|
rand.Seed(seed)
|
|
|
|
c := TestNewCache(t)
|
|
|
|
// save about 5 MiB of data in the cache
|
|
data := rtest.Random(rand.Int(), 5234142)
|
|
id := restic.ID{}
|
|
copy(id[:], data)
|
|
h := backend.Handle{
|
|
Type: restic.PackFile,
|
|
Name: id.String(),
|
|
}
|
|
if err := c.save(h, bytes.NewReader(data)); err != nil {
|
|
t.Fatalf("Save() returned error: %v", err)
|
|
}
|
|
|
|
var tests = []struct {
|
|
offset int64
|
|
length int
|
|
}{
|
|
{0, 0},
|
|
{5, 0},
|
|
{32*1024 + 5, 0},
|
|
{0, 123},
|
|
{0, 64*1024 + 234},
|
|
{100, 5234142 - 100},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(fmt.Sprintf("%v/%v", test.length, test.offset), func(t *testing.T) {
|
|
rd, inCache, err := c.load(h, test.length, test.offset)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rtest.Equals(t, true, inCache, "expected inCache flag to be true")
|
|
|
|
buf, err := io.ReadAll(rd)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if err = rd.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
o := int(test.offset)
|
|
l := test.length
|
|
if test.length == 0 {
|
|
l = len(data) - o
|
|
}
|
|
|
|
if l > len(data)-o {
|
|
l = len(data) - o
|
|
}
|
|
|
|
if len(buf) != l {
|
|
t.Fatalf("wrong number of bytes returned: want %d, got %d", l, len(buf))
|
|
}
|
|
|
|
if !bytes.Equal(buf, data[o:o+l]) {
|
|
t.Fatalf("wrong data returned, want:\n %02x\ngot:\n %02x", data[o:o+16], buf[:16])
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// Simulate multiple processes writing to a cache, using goroutines.
|
|
//
|
|
// The possibility of sharing a cache between multiple concurrent restic
|
|
// processes isn't guaranteed in the docs and doesn't always work on Windows, hence the
|
|
// check on GOOS. Cache sharing is considered a "nice to have" on POSIX, for now.
|
|
//
|
|
// The cache first creates a temporary file and then renames it to its final name.
|
|
// On Windows renaming internally creates a file handle with a shareMode which
|
|
// includes FILE_SHARE_DELETE. The Go runtime opens files without FILE_SHARE_DELETE,
|
|
// thus Open(fn) will fail until the file handle used for renaming was closed.
|
|
// See https://devblogs.microsoft.com/oldnewthing/20211022-00/?p=105822
|
|
// for hints on how to fix this properly.
|
|
func TestFileSaveConcurrent(t *testing.T) {
|
|
if runtime.GOOS == "windows" {
|
|
t.Skip("may not work due to FILE_SHARE_DELETE issue")
|
|
}
|
|
|
|
const nproc = 40
|
|
|
|
var (
|
|
c = TestNewCache(t)
|
|
data = rtest.Random(1, 10000)
|
|
g errgroup.Group
|
|
id restic.ID
|
|
)
|
|
rand.Read(id[:])
|
|
|
|
h := backend.Handle{
|
|
Type: restic.PackFile,
|
|
Name: id.String(),
|
|
}
|
|
|
|
for i := 0; i < nproc/2; i++ {
|
|
g.Go(func() error { return c.save(h, bytes.NewReader(data)) })
|
|
|
|
// Can't use load because only the main goroutine may call t.Fatal.
|
|
g.Go(func() error {
|
|
// The timing is hard to get right, but the main thing we want to
|
|
// ensure is ENOENT or nil error.
|
|
time.Sleep(time.Duration(100+rand.Intn(200)) * time.Millisecond)
|
|
|
|
f, _, err := c.load(h, 0, 0)
|
|
t.Logf("Load error: %v", err)
|
|
switch {
|
|
case err == nil:
|
|
case errors.Is(err, os.ErrNotExist):
|
|
return nil
|
|
default:
|
|
return err
|
|
}
|
|
defer func() { _ = f.Close() }()
|
|
|
|
read, err := io.ReadAll(f)
|
|
if err == nil && !bytes.Equal(read, data) {
|
|
err = errors.New("mismatch between Save and Load")
|
|
}
|
|
return err
|
|
})
|
|
}
|
|
|
|
rtest.OK(t, g.Wait())
|
|
saved := load(t, c, h)
|
|
rtest.Equals(t, data, saved)
|
|
}
|
|
|
|
func TestFileSaveAfterDamage(t *testing.T) {
|
|
c := TestNewCache(t)
|
|
rtest.OK(t, fs.RemoveAll(c.path))
|
|
|
|
// save a few bytes of data in the cache
|
|
data := rtest.Random(123456789, 42)
|
|
id := restic.Hash(data)
|
|
h := backend.Handle{
|
|
Type: restic.PackFile,
|
|
Name: id.String(),
|
|
}
|
|
if err := c.save(h, bytes.NewReader(data)); err == nil {
|
|
t.Fatal("Missing error when saving to deleted cache directory")
|
|
}
|
|
}
|