From 6f76a6db6620fbe1ab429071c694bfffd8784620 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Thu, 16 Mar 2017 21:50:26 +0100 Subject: [PATCH 1/3] rest: Make backend honor the REST protocol --- src/cmds/restic/global.go | 2 +- src/restic/backend/rest/rest.go | 42 +++++++++++++++++++++++++++- src/restic/backend/rest/rest_test.go | 18 +----------- src/restic/backend/test/tests.go | 8 +++--- 4 files changed, 47 insertions(+), 23 deletions(-) diff --git a/src/cmds/restic/global.go b/src/cmds/restic/global.go index 4acd79069..ae3b59ecb 100644 --- a/src/cmds/restic/global.go +++ b/src/cmds/restic/global.go @@ -379,7 +379,7 @@ func create(s string) (restic.Backend, error) { debug.Log("create s3 repository at %#v", loc.Config) return s3.Open(cfg) case "rest": - return rest.Open(loc.Config.(rest.Config)) + return rest.Create(loc.Config.(rest.Config)) } debug.Log("invalid repository scheme: %v", s) diff --git a/src/restic/backend/rest/rest.go b/src/restic/backend/rest/rest.go index 2c133872b..63b6a8094 100644 --- a/src/restic/backend/rest/rest.go +++ b/src/restic/backend/rest/rest.go @@ -69,6 +69,45 @@ func Open(cfg Config) (restic.Backend, error) { return &restBackend{url: cfg.URL, connChan: connChan, client: client}, nil } +// Create creates a new REST on server configured in config. +func Create(cfg Config) (restic.Backend, error) { + be, err := Open(cfg) + if err != nil { + return nil, err + } + + _, err = be.Stat(restic.Handle{Type: restic.ConfigFile}) + if err == nil { + return nil, errors.Fatal("config file already exists") + } + + url := *cfg.URL + values := url.Query() + values.Set("create", "true") + url.RawQuery = values.Encode() + + resp, err := http.Post(url.String(), "binary/octet-stream", strings.NewReader("")) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, errors.Fatalf("server response unexpected: %v (%v)", resp.Status, resp.StatusCode) + } + + _, err = io.Copy(ioutil.Discard, resp.Body) + if err != nil { + return nil, err + } + + err = resp.Body.Close() + if err != nil { + return nil, err + } + + return be, nil +} + // Location returns this backend's location (the server's URL). func (b *restBackend) Location() string { return b.url.String() @@ -103,6 +142,7 @@ func (b *restBackend) Save(h restic.Handle, rd io.Reader) (err error) { return errors.Wrap(err, "client.Post") } + // fmt.Printf("status is %v (%v)\n", resp.Status, resp.StatusCode) if resp.StatusCode != 200 { return errors.Errorf("unexpected HTTP response code %v", resp.StatusCode) } @@ -222,7 +262,7 @@ func (b *restBackend) Remove(h restic.Handle) error { } if resp.StatusCode != 200 { - return errors.New("blob not removed") + return errors.Errorf("blob not removed, server response: %v (%v)", resp.Status, resp.StatusCode) } io.Copy(ioutil.Discard, resp.Body) diff --git a/src/restic/backend/rest/rest_test.go b/src/restic/backend/rest/rest_test.go index af7154ebf..f6f7676ef 100644 --- a/src/restic/backend/rest/rest_test.go +++ b/src/restic/backend/rest/rest_test.go @@ -6,8 +6,6 @@ import ( "os" "restic" - "restic/errors" - "restic/backend/rest" "restic/backend/test" . "restic/test" @@ -32,21 +30,7 @@ func init() { } test.CreateFn = func() (restic.Backend, error) { - be, err := rest.Open(cfg) - if err != nil { - return nil, err - } - - exists, err := be.Test(restic.Handle{Type: restic.ConfigFile, Name: ""}) - if err != nil { - return nil, err - } - - if exists { - return nil, errors.New("config already exists") - } - - return be, nil + return rest.Create(cfg) } test.OpenFn = func() (restic.Backend, error) { diff --git a/src/restic/backend/test/tests.go b/src/restic/backend/test/tests.go index 5c11d2e4b..f2e6b2820 100644 --- a/src/restic/backend/test/tests.go +++ b/src/restic/backend/test/tests.go @@ -375,7 +375,7 @@ var filenameTests = []struct { data string }{ {"1dfc6bc0f06cb255889e9ea7860a5753e8eb9665c9a96627971171b444e3113e", "x"}, - {"foobar", "foobar"}, + {"f00b4r", "foobar"}, { "1dfc6bc0f06cb255889e9ea7860a5753e8eb9665c9a96627971171b444e3113e4bf8f2d9144cc5420a80f04a4880ad6155fc58903a4fb6457c476c43541dcaa6-5", "foobar content of data blob", @@ -500,11 +500,11 @@ func TestBackend(t testing.TB) { ts := testStrings[0] // create blob - err := b.Save(restic.Handle{Type: tpe, Name: ts.id}, strings.NewReader(ts.data)) - test.Assert(t, err != nil, "expected error, got %v", err) + h := restic.Handle{Type: tpe, Name: ts.id} + err := b.Save(h, strings.NewReader(ts.data)) + test.Assert(t, err != nil, "expected error for %v, got %v", h, err) // remove and recreate - h := restic.Handle{Type: tpe, Name: ts.id} err = b.Remove(h) test.OK(t, err) From 990b0f1c1585bb671c2c17925c183eecf221e03f Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Thu, 16 Mar 2017 21:57:32 +0100 Subject: [PATCH 2/3] Travis: Run rest-server for integration tests --- run_integration_tests.go | 66 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/run_integration_tests.go b/run_integration_tests.go index 8c03ed8f4..09106b3f3 100644 --- a/run_integration_tests.go +++ b/run_integration_tests.go @@ -27,6 +27,7 @@ var ForbiddenImports = map[string]bool{ var runCrossCompile = flag.Bool("cross-compile", true, "run cross compilation tests") var minioServer = flag.String("minio", "", "path to the minio server binary") +var restServer = flag.String("rest", "", "path to the rest-server binary") var debug = flag.Bool("debug", false, "output debug messages") var minioServerEnv = map[string]string{ @@ -35,9 +36,10 @@ var minioServerEnv = map[string]string{ } var minioEnv = map[string]string{ - "RESTIC_TEST_S3_SERVER": "http://127.0.0.1:9000", - "AWS_ACCESS_KEY_ID": "KEBIYDZ87HCIH5D17YCN", - "AWS_SECRET_ACCESS_KEY": "bVX1KhipSBPopEfmhc7rGz8ooxx27xdJ7Gkh1mVe", + "RESTIC_TEST_S3_SERVER": "http://127.0.0.1:9000", + "RESTIC_TEST_REST_SERVER": "http://127.0.0.1:8000", + "AWS_ACCESS_KEY_ID": "KEBIYDZ87HCIH5D17YCN", + "AWS_SECRET_ACCESS_KEY": "bVX1KhipSBPopEfmhc7rGz8ooxx27xdJ7Gkh1mVe", } func init() { @@ -54,11 +56,15 @@ type CIEnvironment interface { // TravisEnvironment is the environment in which Travis tests run. type TravisEnvironment struct { goxOSArch []string - minio string + minio string minioSrv *Background minioTempdir string + rest string + restSrv *Background + restTempdir string + env map[string]string } @@ -118,7 +124,7 @@ func (env *TravisEnvironment) runMinio() error { dir, err := ioutil.TempDir("", "minio-root") if err != nil { - return fmt.Errorf("running minio server failed: %v", err) + return fmt.Errorf("TempDir: %v", err) } env.minioSrv, err = StartBackgroundCommand(minioServerEnv, env.minio, @@ -142,6 +148,29 @@ func (env *TravisEnvironment) runMinio() error { return nil } +func (env *TravisEnvironment) runRESTServer() error { + if env.rest == "" { + return nil + } + + // start rest server + msg("starting rest server at %s", env.rest) + + dir, err := ioutil.TempDir("", "rest-server-root") + if err != nil { + return fmt.Errorf("TempDir: %v", err) + } + + env.restSrv, err = StartBackgroundCommand(map[string]string{}, env.rest, + "--path", dir) + if err != nil { + return fmt.Errorf("error running rest server: %v", err) + } + + env.restTempdir = dir + return nil +} + // Prepare installs dependencies and starts services in order to run the tests. func (env *TravisEnvironment) Prepare() error { env.env = make(map[string]string) @@ -151,6 +180,7 @@ func (env *TravisEnvironment) Prepare() error { for _, pkg := range []string{ "golang.org/x/tools/cmd/cover", "github.com/pierrre/gotestcover", + "github.com/restic/rest-server", } { err := run("go", "get", pkg) if err != nil { @@ -164,6 +194,9 @@ func (env *TravisEnvironment) Prepare() error { if err := env.runMinio(); err != nil { return err } + if err := env.runRESTServer(); err != nil { + return err + } if *runCrossCompile && !(runtime.Version() < "go1.7") { // only test cross compilation on linux with Travis @@ -227,6 +260,29 @@ func (env *TravisEnvironment) Teardown() error { } } + if env.restSrv != nil { + msg("stopping rest-server\n") + + if env.restSrv.Cmd.ProcessState == nil { + err := env.restSrv.Cmd.Process.Kill() + if err != nil { + fmt.Fprintf(os.Stderr, "error killing rest-server process: %v", err) + } + } else { + result := <-env.restSrv.Result + if result.Error != nil { + msg("rest-server returned error: %v\n", result.Error) + msg("stdout: %s\n", result.Stdout) + msg("stderr: %s\n", result.Stderr) + } + } + + err := os.RemoveAll(env.restTempdir) + if err != nil { + msg("error removing rest-server tempdir %v: %v\n", env.restTempdir, err) + } + } + return nil } From 280f05b1741b422752b486eeedfb1d95695eff4a Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 18 Mar 2017 11:11:31 +0100 Subject: [PATCH 3/3] Run rest-server --- run_integration_tests.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/run_integration_tests.go b/run_integration_tests.go index 09106b3f3..53207550c 100644 --- a/run_integration_tests.go +++ b/run_integration_tests.go @@ -188,6 +188,8 @@ func (env *TravisEnvironment) Prepare() error { } } + env.rest = filepath.Join(os.Getenv("GOPATH"), "bin", "rest-server") + if err := env.getMinio(); err != nil { return err }