diff --git a/src/restic/backend/s3/config.go b/src/restic/backend/s3/config.go index 3ae6038ab..6e8e9f2f6 100644 --- a/src/restic/backend/s3/config.go +++ b/src/restic/backend/s3/config.go @@ -25,7 +25,7 @@ type Config struct { // NewConfig returns a new Config with the default values filled in. func NewConfig() Config { return Config{ - Connections: 20, + Connections: 5, } } diff --git a/src/restic/backend/s3/config_test.go b/src/restic/backend/s3/config_test.go index 67611f3cc..8e5f3a420 100644 --- a/src/restic/backend/s3/config_test.go +++ b/src/restic/backend/s3/config_test.go @@ -10,89 +10,89 @@ var configTests = []struct { Endpoint: "eu-central-1", Bucket: "bucketname", Prefix: "restic", - Connections: 20, + Connections: 5, }}, {"s3://eu-central-1/bucketname/", Config{ Endpoint: "eu-central-1", Bucket: "bucketname", Prefix: "restic", - Connections: 20, + Connections: 5, }}, {"s3://eu-central-1/bucketname/prefix/directory", Config{ Endpoint: "eu-central-1", Bucket: "bucketname", Prefix: "prefix/directory", - Connections: 20, + Connections: 5, }}, {"s3://eu-central-1/bucketname/prefix/directory/", Config{ Endpoint: "eu-central-1", Bucket: "bucketname", Prefix: "prefix/directory", - Connections: 20, + Connections: 5, }}, {"s3:eu-central-1/foobar", Config{ Endpoint: "eu-central-1", Bucket: "foobar", Prefix: "restic", - Connections: 20, + Connections: 5, }}, {"s3:eu-central-1/foobar/", Config{ Endpoint: "eu-central-1", Bucket: "foobar", Prefix: "restic", - Connections: 20, + Connections: 5, }}, {"s3:eu-central-1/foobar/prefix/directory", Config{ Endpoint: "eu-central-1", Bucket: "foobar", Prefix: "prefix/directory", - Connections: 20, + Connections: 5, }}, {"s3:eu-central-1/foobar/prefix/directory/", Config{ Endpoint: "eu-central-1", Bucket: "foobar", Prefix: "prefix/directory", - Connections: 20, + Connections: 5, }}, {"s3:https://hostname:9999/foobar", Config{ Endpoint: "hostname:9999", Bucket: "foobar", Prefix: "restic", - Connections: 20, + Connections: 5, }}, {"s3:https://hostname:9999/foobar/", Config{ Endpoint: "hostname:9999", Bucket: "foobar", Prefix: "restic", - Connections: 20, + Connections: 5, }}, {"s3:http://hostname:9999/foobar", Config{ Endpoint: "hostname:9999", Bucket: "foobar", Prefix: "restic", UseHTTP: true, - Connections: 20, + Connections: 5, }}, {"s3:http://hostname:9999/foobar/", Config{ Endpoint: "hostname:9999", Bucket: "foobar", Prefix: "restic", UseHTTP: true, - Connections: 20, + Connections: 5, }}, {"s3:http://hostname:9999/bucket/prefix/directory", Config{ Endpoint: "hostname:9999", Bucket: "bucket", Prefix: "prefix/directory", UseHTTP: true, - Connections: 20, + Connections: 5, }}, {"s3:http://hostname:9999/bucket/prefix/directory/", Config{ Endpoint: "hostname:9999", Bucket: "bucket", Prefix: "prefix/directory", UseHTTP: true, - Connections: 20, + Connections: 5, }}, } diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index beac864b5..6db2e7da9 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -157,63 +157,15 @@ func (be *Backend) Path() string { return be.prefix } -// getRemainingSize returns number of bytes remaining. If it is not possible to -// determine the size, panic() is called. -func getRemainingSize(rd io.Reader) (size int64, err error) { - type Sizer interface { - Size() int64 - } - - type Lenner interface { - Len() int - } - - if r, ok := rd.(Lenner); ok { - size = int64(r.Len()) - } else if r, ok := rd.(Sizer); ok { - size = r.Size() - } else if f, ok := rd.(*os.File); ok { - fi, err := f.Stat() - if err != nil { - return 0, err - } - - pos, err := f.Seek(0, io.SeekCurrent) - if err != nil { - return 0, err - } - - size = fi.Size() - pos - } else { - panic(fmt.Sprintf("Save() got passed a reader without a method to determine the data size, type is %T", rd)) - } - return size, nil -} - -// preventCloser wraps an io.Reader to run a function instead of the original Close() function. -type preventCloser struct { - io.Reader - f func() -} - -func (wr preventCloser) Close() error { - wr.f() - return nil -} - // Save stores data in the backend at the handle. func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) { + debug.Log("Save %v", h) + if err := h.Valid(); err != nil { return err } objName := be.Filename(h) - size, err := getRemainingSize(rd) - if err != nil { - return err - } - - debug.Log("Save %v at %v (%d bytes)", h, objName, size) // Check key does not already exist _, err = be.client.StatObject(be.bucketname, objName) @@ -222,23 +174,12 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err return errors.New("key already exists") } - be.sem.GetToken() - - // wrap the reader so that net/http client cannot close the reader, return - // the token instead. - rd = preventCloser{ - Reader: rd, - f: func() { - debug.Log("Close()") - }, - } - debug.Log("PutObject(%v, %v)", be.bucketname, objName) - coreClient := minio.Core{Client: be.client} - info, err := coreClient.PutObject(be.bucketname, objName, size, rd, nil, nil, nil) - + be.sem.GetToken() + n, err := be.client.PutObject(be.bucketname, objName, rd, "application/octet-stream") be.sem.ReleaseToken() - debug.Log("%v -> %v bytes, err %#v: %v", objName, info.Size, err, err) + + debug.Log("%v -> %v bytes, err %#v: %v", objName, n, err, err) return errors.Wrap(err, "client.PutObject") } diff --git a/src/restic/backend/s3/s3_internal_test.go b/src/restic/backend/s3/s3_internal_test.go deleted file mode 100644 index ce6c96036..000000000 --- a/src/restic/backend/s3/s3_internal_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package s3 - -import ( - "bytes" - "io" - "io/ioutil" - "os" - "restic/test" - "testing" -) - -func writeFile(t testing.TB, data []byte, offset int64) *os.File { - tempfile, err := ioutil.TempFile("", "restic-test-") - if err != nil { - t.Fatal(err) - } - - if _, err = tempfile.Write(data); err != nil { - t.Fatal(err) - } - - if _, err = tempfile.Seek(offset, io.SeekStart); err != nil { - t.Fatal(err) - } - - return tempfile -} - -func TestGetRemainingSize(t *testing.T) { - length := 18 * 1123 - partialRead := 1005 - - data := test.Random(23, length) - - partReader := bytes.NewReader(data) - buf := make([]byte, partialRead) - _, _ = io.ReadFull(partReader, buf) - - partFileReader := writeFile(t, data, int64(partialRead)) - defer func() { - if err := partFileReader.Close(); err != nil { - t.Fatal(err) - } - - if err := os.Remove(partFileReader.Name()); err != nil { - t.Fatal(err) - } - }() - - var tests = []struct { - io.Reader - size int64 - }{ - {bytes.NewReader([]byte("foobar test")), 11}, - {partReader, int64(length - partialRead)}, - {partFileReader, int64(length - partialRead)}, - } - - for _, test := range tests { - t.Run("", func(t *testing.T) { - size, err := getRemainingSize(test.Reader) - if err != nil { - t.Fatal(err) - } - - if size != test.size { - t.Fatalf("invalid size returned, want %v, got %v", test.size, size) - } - }) - } -}