Cancel current command if cache becomes unusable

If the cache suddenly disappears, the current command will now fail.
This commit is contained in:
Michael Eischer 2023-02-10 22:39:40 +01:00
parent 22562d2132
commit 78a1757e5a
3 changed files with 44 additions and 16 deletions

View File

@ -0,0 +1,7 @@
Enhancement: Cancel current command if cache becomes unusable
If the cache directory was removed or ran out of space while restic was
running, this caused further caching attempts to fail and drastically slow down
the command execution. Now, the currently running command is canceled instead.
https://github.com/restic/restic/pull/4166

View File

@ -83,7 +83,7 @@ func (b *Backend) Save(ctx context.Context, h restic.Handle, rd restic.RewindRea
if err != nil {
debug.Log("unable to save %v to cache: %v", h, err)
_ = b.Cache.remove(h)
return nil
return err
}
return nil
@ -106,11 +106,19 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error {
return nil
}
defer func() {
// signal other waiting goroutines that the file may now be cached
close(finish)
// remove the finish channel from the map
b.inProgressMutex.Lock()
delete(b.inProgress, h)
b.inProgressMutex.Unlock()
}()
// test again, maybe the file was cached in the meantime
if !b.Cache.Has(h) {
// nope, it's still not in the cache, pull it from the repo and save it
err := b.Backend.Load(ctx, h, 0, 0, func(rd io.Reader) error {
return b.Cache.Save(h, rd)
})
@ -118,16 +126,9 @@ func (b *Backend) cacheFile(ctx context.Context, h restic.Handle) error {
// try to remove from the cache, ignore errors
_ = b.Cache.remove(h)
}
return err
}
// signal other waiting goroutines that the file may now be cached
close(finish)
// remove the finish channel from the map
b.inProgressMutex.Lock()
delete(b.inProgress, h)
b.inProgressMutex.Unlock()
return nil
}
@ -178,11 +179,13 @@ func (b *Backend) Load(ctx context.Context, h restic.Handle, length int, offset
debug.Log("auto-store %v in the cache", h)
err = b.cacheFile(ctx, h)
if err == nil {
inCache, err = b.loadFromCache(ctx, h, length, offset, consumer)
if inCache {
return err
}
if err != nil {
return err
}
inCache, err = b.loadFromCache(ctx, h, length, offset, consumer)
if inCache {
return err
}
debug.Log("error caching %v: %v, falling back to backend", h, err)

View File

@ -11,8 +11,10 @@ import (
"time"
"github.com/restic/restic/internal/errors"
"github.com/restic/restic/internal/fs"
"github.com/restic/restic/internal/restic"
"github.com/restic/restic/internal/test"
rtest "github.com/restic/restic/internal/test"
"golang.org/x/sync/errgroup"
)
@ -266,3 +268,19 @@ func TestFileSaveConcurrent(t *testing.T) {
saved := load(t, c, h)
test.Equals(t, data, saved)
}
func TestFileSaveAfterDamage(t *testing.T) {
c := TestNewCache(t)
rtest.OK(t, fs.RemoveAll(c.path))
// save a few bytes of data in the cache
data := test.Random(123456789, 42)
id := restic.Hash(data)
h := restic.Handle{
Type: restic.PackFile,
Name: id.String(),
}
if err := c.Save(h, bytes.NewReader(data)); err == nil {
t.Fatal("Missing error when saving to deleted cache directory")
}
}