From b5062959c81eaf26375ec5642f8418db4666de15 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 17 Feb 2018 22:39:18 +0100 Subject: [PATCH] backend: Relax requirement for new files Before, all backend implementations were required to return an error if the file that is to be written already exists in the backend. For most backends, that means making a request (e.g. via HTTP) and returning an error when the file already exists. This is not accurate, the file could have been created between the HTTP request testing for it, and when writing starts. In addition, apart from the `config` file in the repo, all other file names have pseudo-random names with a very very low probability of a collision. And even if a file name is written again, the way the restic repo is structured this just means that the same content is placed there again. Which is not a problem, just not very efficient. So, this commit relaxes the requirement to return an error when the file in the backend already exists, which allows reducing the number of API requests and thereby the latency for remote backends. --- internal/backend/azure/azure.go | 10 ---------- internal/backend/b2/b2.go | 6 ------ internal/backend/gs/gs.go | 7 ------- internal/backend/s3/s3.go | 7 ------- internal/backend/swift/swift.go | 13 ------------- internal/backend/test/tests.go | 6 +----- 6 files changed, 1 insertion(+), 48 deletions(-) diff --git a/internal/backend/azure/azure.go b/internal/backend/azure/azure.go index ed401c868..2b6697dc7 100644 --- a/internal/backend/azure/azure.go +++ b/internal/backend/azure/azure.go @@ -135,16 +135,6 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err debug.Log("Save %v at %v", h, objName) - // Check key does not already exist - found, err := be.container.GetBlobReference(objName).Exists() - if err != nil { - return errors.Wrap(err, "GetBlobReference().Exists()") - } - if found { - debug.Log("%v already exists", h) - return errors.New("key already exists") - } - be.sem.GetToken() // wrap the reader so that net/http client cannot close the reader, return diff --git a/internal/backend/b2/b2.go b/internal/backend/b2/b2.go index ec5bf8d9c..638e69255 100644 --- a/internal/backend/b2/b2.go +++ b/internal/backend/b2/b2.go @@ -196,12 +196,6 @@ func (be *b2Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) er debug.Log("Save %v, name %v", h, name) obj := be.bucket.Object(name) - _, err := obj.Attrs(ctx) - if err == nil { - debug.Log(" %v already exists", h) - return errors.New("key already exists") - } - w := obj.NewWriter(ctx) n, err := io.Copy(w, rd) debug.Log(" saved %d bytes, err %v", n, err) diff --git a/internal/backend/gs/gs.go b/internal/backend/gs/gs.go index d273d3e71..f963b3f88 100644 --- a/internal/backend/gs/gs.go +++ b/internal/backend/gs/gs.go @@ -218,13 +218,6 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err be.sem.GetToken() - // Check key does not already exist - if _, err := be.service.Objects.Get(be.bucketName, objName).Do(); err == nil { - debug.Log("%v already exists", h) - be.sem.ReleaseToken() - return errors.New("key already exists") - } - debug.Log("InsertObject(%v, %v)", be.bucketName, objName) // Set chunk size to zero to disable resumable uploads. diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index b33e76f64..9e57e32e0 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -235,13 +235,6 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err be.sem.GetToken() defer be.sem.ReleaseToken() - // Check key does not already exist - _, err = be.client.StatObject(be.cfg.Bucket, objName, minio.StatObjectOptions{}) - if err == nil { - debug.Log("%v already exists", h) - return errors.New("key already exists") - } - var size int64 = -1 type lenner interface { diff --git a/internal/backend/swift/swift.go b/internal/backend/swift/swift.go index 27df0d55a..bce0d55a8 100644 --- a/internal/backend/swift/swift.go +++ b/internal/backend/swift/swift.go @@ -165,19 +165,6 @@ func (be *beSwift) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err be.sem.GetToken() defer be.sem.ReleaseToken() - // Check key does not already exist - switch _, _, err = be.conn.Object(be.container, objName); err { - case nil: - debug.Log("%v already exists", h) - return errors.New("key already exists") - - case swift.ObjectNotFound: - // Ok, that's what we want - - default: - return errors.Wrap(err, "conn.Object") - } - encoding := "binary/octet-stream" debug.Log("PutObject(%v, %v, %v)", be.container, objName, encoding) diff --git a/internal/backend/test/tests.go b/internal/backend/test/tests.go index f8b321a74..2951d15f6 100644 --- a/internal/backend/test/tests.go +++ b/internal/backend/test/tests.go @@ -793,14 +793,10 @@ func (s *Suite) TestBackend(t *testing.T) { // test adding the first file again ts := testStrings[0] - - // create blob h := restic.Handle{Type: tpe, Name: ts.id} - err := b.Save(context.TODO(), h, strings.NewReader(ts.data)) - test.Assert(t, err != nil, "backend has allowed overwrite of existing blob: expected error for %v, got %v", h, err) // remove and recreate - err = s.delayedRemove(t, b, h) + err := s.delayedRemove(t, b, h) test.OK(t, err) // test that the blob is gone