Merge pull request #1594 from restic/gcs-use-http-transport

GS: Use generic http transport

Gopkg.lock

@@ -231,6 +231,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "a7d099b3ce195ffc37adedb05a4386be38e6158925a1c0fe579efdc20fa11f6a"
+  inputs-digest = "336ac5c261c174cac89f9a7102b493f08edfbd51fd61d1673d1d2ec4132d80ab"
   solver-name = "gps-cdcl"
   solver-version = 1

changelog/0.8.2/pull-1594

@@ -0,0 +1,7 @@
+Bugfix: Google Cloud Storage: Use generic HTTP transport
+
+It was discovered that the Google Cloud Storage backend did not use the generic
+HTTP transport, so things such as bandwidth limiting with `--limit-upload` did
+not work. This is resolved now.
+
+https://github.com/restic/restic/pull/1594

cmd/restic/global.go

@@ -567,7 +567,7 @@ func open(s string, gopts GlobalOptions, opts options.Options) (restic.Backend,
 	case "s3":
 		be, err = s3.Open(cfg.(s3.Config), rt)
 	case "gs":
-		be, err = gs.Open(cfg.(gs.Config))
+		be, err = gs.Open(cfg.(gs.Config), rt)
 	case "azure":
 		be, err = azure.Open(cfg.(azure.Config), rt)
 	case "swift":
@@ -628,7 +628,7 @@ func create(s string, opts options.Options) (restic.Backend, error) {
 	case "s3":
 		return s3.Create(cfg.(s3.Config), rt)
 	case "gs":
-		return gs.Create(cfg.(gs.Config))
+		return gs.Create(cfg.(gs.Config), rt)
 	case "azure":
 		return azure.Create(cfg.(azure.Config), rt)
 	case "swift":

internal/backend/gs/gs.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net/http"
 	"os"
 	"path"
 	"strings"
@@ -16,6 +17,7 @@ import (
 
 	"io/ioutil"
 
+	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
 	"google.golang.org/api/googleapi"
 	storage "google.golang.org/api/storage/v1"
@@ -41,7 +43,7 @@ type Backend struct {
 // Ensure that *Backend implements restic.Backend.
 var _ restic.Backend = &Backend{}
 
-func getStorageService(jsonKeyPath string) (*storage.Service, error) {
+func getStorageService(jsonKeyPath string, rt http.RoundTripper) (*storage.Service, error) {
 
 	raw, err := ioutil.ReadFile(jsonKeyPath)
 	if err != nil {
@@ -53,8 +55,18 @@ func getStorageService(jsonKeyPath string) (*storage.Service, error) {
 		return nil, err
 	}
 
-	client := conf.Client(context.TODO())
+	// create a new HTTP client
+	httpClient := &http.Client{
+		Transport: rt,
+	}
 
+	// create a now context with the HTTP client stored at the oauth2.HTTPClient key
+	ctx := context.WithValue(context.Background(), oauth2.HTTPClient, httpClient)
+
+	// then pass this context to Client(), which returns a new HTTP client
+	client := conf.Client(ctx)
+
+	// that we can then pass to New()
 	service, err := storage.New(client)
 	if err != nil {
 		return nil, err
@@ -65,10 +77,10 @@ func getStorageService(jsonKeyPath string) (*storage.Service, error) {
 
 const defaultListMaxItems = 1000
 
-func open(cfg Config) (*Backend, error) {
+func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
 	debug.Log("open, config %#v", cfg)
 
-	service, err := getStorageService(cfg.JSONKeyPath)
+	service, err := getStorageService(cfg.JSONKeyPath, rt)
 	if err != nil {
 		return nil, errors.Wrap(err, "getStorageService")
 	}
@@ -95,8 +107,8 @@ func open(cfg Config) (*Backend, error) {
 }
 
 // Open opens the gs backend at the specified bucket.
-func Open(cfg Config) (restic.Backend, error) {
-	return open(cfg)
+func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
+	return open(cfg, rt)
 }
 
 // Create opens the gs backend at the specified bucket and attempts to creates
@@ -104,8 +116,8 @@ func Open(cfg Config) (restic.Backend, error) {
 //
 // The service account must have the "storage.buckets.create" permission to
 // create a bucket the does not yet exist.
-func Create(cfg Config) (restic.Backend, error) {
-	be, err := open(cfg)
+func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
+	be, err := open(cfg, rt)
 	if err != nil {
 		return nil, errors.Wrap(err, "open")
 	}

internal/backend/gs/gs_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/backend/gs"
 	"github.com/restic/restic/internal/backend/test"
 	"github.com/restic/restic/internal/errors"
@@ -15,6 +16,11 @@ import (
 )
 
 func newGSTestSuite(t testing.TB) *test.Suite {
+	tr, err := backend.Transport(backend.TransportOptions{})
+	if err != nil {
+		t.Fatalf("cannot create transport for tests: %v", err)
+	}
+
 	return &test.Suite{
 		// do not use excessive data
 		MinimalData: true,
@@ -37,7 +43,7 @@ func newGSTestSuite(t testing.TB) *test.Suite {
 		Create: func(config interface{}) (restic.Backend, error) {
 			cfg := config.(gs.Config)
 
-			be, err := gs.Create(cfg)
+			be, err := gs.Create(cfg, tr)
 			if err != nil {
 				return nil, err
 			}
@@ -57,14 +63,14 @@ func newGSTestSuite(t testing.TB) *test.Suite {
 		// OpenFn is a function that opens a previously created temporary repository.
 		Open: func(config interface{}) (restic.Backend, error) {
 			cfg := config.(gs.Config)
-			return gs.Open(cfg)
+			return gs.Open(cfg, tr)
 		},
 
 		// CleanupFn removes data created during the tests.
 		Cleanup: func(config interface{}) error {
 			cfg := config.(gs.Config)
 
-			be, err := gs.Open(cfg)
+			be, err := gs.Open(cfg, tr)
 			if err != nil {
 				return err
 			}