From b942f61272af45c5a5c02d2bc8265a4065c65304 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 17 Apr 2017 19:18:58 +0200 Subject: [PATCH 1/3] s3: Reduce connection limit, prevents timeouts --- src/restic/backend/s3/s3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index f59a92a15..18548ef40 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -17,7 +17,7 @@ import ( "restic/debug" ) -const connLimit = 40 +const connLimit = 10 // s3 is a backend which stores the data on an S3 endpoint. type s3 struct { From 16fd1c2352ed9326ed6eadfd61ae4e461e219371 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 17 Apr 2017 20:06:06 +0200 Subject: [PATCH 2/3] s3: Correct layout handle url/path --- src/restic/backend/layout_s3.go | 20 ++++++++++++++++++-- src/restic/backend/layout_test.go | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/restic/backend/layout_s3.go b/src/restic/backend/layout_s3.go index 9db8ae7f3..91273d63a 100644 --- a/src/restic/backend/layout_s3.go +++ b/src/restic/backend/layout_s3.go @@ -18,13 +18,29 @@ var s3LayoutPaths = map[restic.FileType]string{ restic.KeyFile: "key", } +// join calls Join with the first empty elements removed. +func (l *S3Layout) join(url string, items ...string) string { + for len(items) > 0 && items[0] == "" { + items = items[1:] + } + + path := l.Join(items...) + if path == "" || path[0] != '/' { + if url != "" && url[len(url)-1] != '/' { + url += "/" + } + } + + return url + path +} + // Dirname returns the directory path for a given file type and name. func (l *S3Layout) Dirname(h restic.Handle) string { if h.Type == restic.ConfigFile { return l.URL + l.Join(l.Path, "/") } - return l.URL + l.Join(l.Path, "/", s3LayoutPaths[h.Type]) + "/" + return l.join(l.URL, l.Path, s3LayoutPaths[h.Type]) + "/" } // Filename returns a path to a file, including its name. @@ -35,7 +51,7 @@ func (l *S3Layout) Filename(h restic.Handle) string { name = "config" } - return l.URL + l.Join(l.Path, "/", s3LayoutPaths[h.Type], name) + return l.join(l.URL, l.Path, s3LayoutPaths[h.Type], name) } // Paths returns all directory names diff --git a/src/restic/backend/layout_test.go b/src/restic/backend/layout_test.go index 4fee713cb..606da3241 100644 --- a/src/restic/backend/layout_test.go +++ b/src/restic/backend/layout_test.go @@ -190,6 +190,24 @@ func TestCloudLayoutURLs(t *testing.T) { "https://hostname.foo:1234/prefix/repo/config", "https://hostname.foo:1234/prefix/repo/", }, + { + &S3Layout{URL: "", Path: "", Join: path.Join}, + restic.Handle{Type: restic.DataFile, Name: "foobar"}, + "data/foobar", + "data/", + }, + { + &S3Layout{URL: "", Path: "", Join: path.Join}, + restic.Handle{Type: restic.LockFile, Name: "foobar"}, + "lock/foobar", + "lock/", + }, + { + &S3Layout{URL: "", Path: "/", Join: path.Join}, + restic.Handle{Type: restic.ConfigFile, Name: "foobar"}, + "/config", + "/", + }, } for _, test := range tests { From 28968caf338095e0102a84e45d4a8957dc06b2fd Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Mon, 17 Apr 2017 19:18:47 +0200 Subject: [PATCH 3/3] s3: Correct prefix for layout --- src/restic/backend/s3/s3.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index 18548ef40..2f6db84f5 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -45,7 +45,7 @@ func Open(cfg Config) (restic.Backend, error) { bucketname: cfg.Bucket, prefix: cfg.Prefix, cacheObjSize: make(map[string]int64), - Layout: &backend.S3Layout{URL: cfg.Endpoint, Join: path.Join}, + Layout: &backend.S3Layout{Path: cfg.Prefix, Join: path.Join}, } tr := &http.Transport{MaxIdleConnsPerHost: connLimit} @@ -88,10 +88,10 @@ func (be *s3) Save(h restic.Handle, rd io.Reader) (err error) { return err } - debug.Log("Save %v", h) - objName := be.Filename(h) + debug.Log("Save %v at %v", h, objName) + // Check key does not already exist _, err = be.client.StatObject(be.bucketname, objName) if err == nil { @@ -128,7 +128,7 @@ func (wr wrapReader) Close() error { // given offset. If length is nonzero, only a portion of the file is // returned. rd must be closed after use. func (be *s3) Load(h restic.Handle, length int, offset int64) (io.ReadCloser, error) { - debug.Log("Load %v, length %v, offset %v", h, length, offset) + debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h)) if err := h.Valid(); err != nil { return nil, err } @@ -280,7 +280,7 @@ func (be *s3) Test(h restic.Handle) (bool, error) { func (be *s3) Remove(h restic.Handle) error { objName := be.Filename(h) err := be.client.RemoveObject(be.bucketname, objName) - debug.Log("Remove(%v) -> err %v", h, err) + debug.Log("Remove(%v) at %v -> err %v", h, objName, err) return errors.Wrap(err, "client.RemoveObject") }