From eef73d466db0f3a0377ee5fd1d36d24e910b5c92 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 20 Feb 2016 18:33:06 +0100 Subject: [PATCH] Properly vendor dependencies with gb-vendor --- vendor/manifest | 82 + vendor/src/bazil.org/fuse/.gitattributes | 2 - vendor/src/bazil.org/fuse/.gitignore | 11 - vendor/src/bazil.org/fuse/doc/.gitignore | 4 - .../src/bazil.org/fuse/fs/fstestutil/doc.go | 2 +- .../fuse/fs/fstestutil/record/record.go | 2 +- vendor/src/bazil.org/fuse/fs/serve.go | 2 +- vendor/src/bazil.org/fuse/fuse.go | 2 +- .../src/bazil.org/fuse/fuseutil/fuseutil.go | 2 +- vendor/src/bazil.org/fuse/syscallx/doc.go | 2 +- .../github.com/jessevdk/go-flags/.travis.yml | 35 - vendor/src/github.com/juju/errors/.gitignore | 23 - .../src/github.com/minio/minio-go/.gitignore | 2 - .../src/github.com/minio/minio-go/.travis.yml | 23 - .../minio/minio-go/api_functional_v2_test.go | 1043 +++++++++++ .../minio/minio-go/api_functional_v4_test.go | 1072 +++++++++++ .../minio/minio-go/api_unit_test.go | 583 ++++++ vendor/src/github.com/pkg/sftp/.gitignore | 8 - vendor/src/github.com/pkg/sftp/.travis.yml | 21 - vendor/src/github.com/pkg/sftp/attrs_test.go | 45 + .../sftp/client_integration_darwin_test.go | 46 + .../pkg/sftp/client_integration_linux_test.go | 46 + .../pkg/sftp/client_integration_test.go | 1616 +++++++++++++++++ vendor/src/github.com/pkg/sftp/client_test.go | 99 + .../src/github.com/pkg/sftp/example_test.go | 90 + vendor/src/github.com/pkg/sftp/other_test.go | 5 + vendor/src/github.com/pkg/sftp/packet_test.go | 345 ++++ .../pkg/sftp/server_integration_test.go | 672 +++++++ .../src/github.com/restic/chunker/.travis.yml | 11 - .../src/golang.org/x/crypto/pbkdf2/pbkdf2.go | 2 +- .../golang.org/x/crypto/poly1305/poly1305.go | 2 +- .../src/golang.org/x/crypto/scrypt/scrypt.go | 2 +- .../golang.org/x/crypto/ssh/agent/client.go | 2 +- vendor/src/golang.org/x/crypto/ssh/doc.go | 2 +- .../golang.org/x/crypto/ssh/terminal/util.go | 2 +- .../src/golang.org/x/crypto/ssh/test/doc.go | 2 +- .../golang.org/x/crypto/ssh/testdata/doc.go | 2 +- .../src/golang.org/x/net/context/context.go | 2 +- .../x/net/context/ctxhttp/ctxhttp.go | 2 +- 39 files changed, 5760 insertions(+), 156 deletions(-) create mode 100644 vendor/manifest delete mode 100644 vendor/src/bazil.org/fuse/.gitattributes delete mode 100644 vendor/src/bazil.org/fuse/.gitignore delete mode 100644 vendor/src/bazil.org/fuse/doc/.gitignore delete mode 100644 vendor/src/github.com/jessevdk/go-flags/.travis.yml delete mode 100644 vendor/src/github.com/juju/errors/.gitignore delete mode 100644 vendor/src/github.com/minio/minio-go/.gitignore delete mode 100644 vendor/src/github.com/minio/minio-go/.travis.yml create mode 100644 vendor/src/github.com/minio/minio-go/api_functional_v2_test.go create mode 100644 vendor/src/github.com/minio/minio-go/api_functional_v4_test.go create mode 100644 vendor/src/github.com/minio/minio-go/api_unit_test.go delete mode 100644 vendor/src/github.com/pkg/sftp/.gitignore delete mode 100644 vendor/src/github.com/pkg/sftp/.travis.yml create mode 100644 vendor/src/github.com/pkg/sftp/attrs_test.go create mode 100644 vendor/src/github.com/pkg/sftp/client_integration_darwin_test.go create mode 100644 vendor/src/github.com/pkg/sftp/client_integration_linux_test.go create mode 100644 vendor/src/github.com/pkg/sftp/client_integration_test.go create mode 100644 vendor/src/github.com/pkg/sftp/client_test.go create mode 100644 vendor/src/github.com/pkg/sftp/example_test.go create mode 100644 vendor/src/github.com/pkg/sftp/other_test.go create mode 100644 vendor/src/github.com/pkg/sftp/packet_test.go create mode 100644 vendor/src/github.com/pkg/sftp/server_integration_test.go delete mode 100644 vendor/src/github.com/restic/chunker/.travis.yml diff --git a/vendor/manifest b/vendor/manifest new file mode 100644 index 000000000..6caebea93 --- /dev/null +++ b/vendor/manifest @@ -0,0 +1,82 @@ +{ + "version": 0, + "dependencies": [ + { + "importpath": "bazil.org/fuse", + "repository": "https://github.com/bazil/fuse", + "revision": "18419ee53958df28fcfc9490fe6123bd59e237bb", + "branch": "HEAD" + }, + { + "importpath": "github.com/jessevdk/go-flags", + "repository": "https://github.com/jessevdk/go-flags", + "revision": "1b89bf73cd2c3a911d7b2a279ab085c4a18cf539", + "branch": "HEAD" + }, + { + "importpath": "github.com/juju/errors", + "repository": "https://github.com/juju/errors", + "revision": "4567a5e69fd3130ca0d89f69478e7ac025b67452", + "branch": "HEAD" + }, + { + "importpath": "github.com/kr/fs", + "repository": "https://github.com/kr/fs", + "revision": "2788f0dbd16903de03cb8186e5c7d97b69ad387b", + "branch": "HEAD" + }, + { + "importpath": "github.com/minio/minio-go", + "repository": "https://github.com/minio/minio-go", + "revision": "a4cd3caabd5f9c35ac100110eb60c2b80798f1af", + "branch": "HEAD" + }, + { + "importpath": "github.com/pkg/sftp", + "repository": "https://github.com/pkg/sftp", + "revision": "e84cc8c755ca39b7b64f510fe1fffc1b51f210a5", + "branch": "HEAD" + }, + { + "importpath": "github.com/restic/chunker", + "repository": "https://github.com/restic/chunker", + "revision": "fc45043175c38d59374024a38fb7123c40a64f20", + "branch": "HEAD" + }, + { + "importpath": "golang.org/x/crypto/pbkdf2", + "repository": "https://go.googlesource.com/crypto", + "revision": "cc04154d65fb9296747569b107cfd05380b1ea3e", + "branch": "HEAD", + "path": "/pbkdf2" + }, + { + "importpath": "golang.org/x/crypto/poly1305", + "repository": "https://go.googlesource.com/crypto", + "revision": "cc04154d65fb9296747569b107cfd05380b1ea3e", + "branch": "HEAD", + "path": "/poly1305" + }, + { + "importpath": "golang.org/x/crypto/scrypt", + "repository": "https://go.googlesource.com/crypto", + "revision": "cc04154d65fb9296747569b107cfd05380b1ea3e", + "branch": "HEAD", + "path": "/scrypt" + }, + { + "importpath": "golang.org/x/crypto/ssh", + "repository": "https://go.googlesource.com/crypto", + "revision": "cc04154d65fb9296747569b107cfd05380b1ea3e", + "branch": "HEAD", + "path": "/ssh" + }, + { + "importpath": "golang.org/x/net/context", + "repository": "https://go.googlesource.com/net", + "revision": "7654728e381988afd88e58cabfd6363a5ea91810", + "branch": "HEAD", + "path": "/context" + } + ] +} \ No newline at end of file diff --git a/vendor/src/bazil.org/fuse/.gitattributes b/vendor/src/bazil.org/fuse/.gitattributes deleted file mode 100644 index b65f2a9ff..000000000 --- a/vendor/src/bazil.org/fuse/.gitattributes +++ /dev/null @@ -1,2 +0,0 @@ -*.go filter=gofmt -*.cgo filter=gofmt diff --git a/vendor/src/bazil.org/fuse/.gitignore b/vendor/src/bazil.org/fuse/.gitignore deleted file mode 100644 index 53589948c..000000000 --- a/vendor/src/bazil.org/fuse/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -*~ -.#* -## the next line needs to start with a backslash to avoid looking like -## a comment -\#*# -.*.swp - -*.test - -/clockfs -/hellofs diff --git a/vendor/src/bazil.org/fuse/doc/.gitignore b/vendor/src/bazil.org/fuse/doc/.gitignore deleted file mode 100644 index 6ebe2d170..000000000 --- a/vendor/src/bazil.org/fuse/doc/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/*.seq.svg - -# not ignoring *.seq.png; we want those committed to the repo -# for embedding on Github diff --git a/vendor/src/bazil.org/fuse/fs/fstestutil/doc.go b/vendor/src/bazil.org/fuse/fs/fstestutil/doc.go index d4366cca5..3f729dddc 100644 --- a/vendor/src/bazil.org/fuse/fs/fstestutil/doc.go +++ b/vendor/src/bazil.org/fuse/fs/fstestutil/doc.go @@ -1 +1 @@ -package fstestutil +package fstestutil // import "bazil.org/fuse/fs/fstestutil" diff --git a/vendor/src/bazil.org/fuse/fs/fstestutil/record/record.go b/vendor/src/bazil.org/fuse/fs/fstestutil/record/record.go index 2bf2e9f8b..1b07da5ca 100644 --- a/vendor/src/bazil.org/fuse/fs/fstestutil/record/record.go +++ b/vendor/src/bazil.org/fuse/fs/fstestutil/record/record.go @@ -1,4 +1,4 @@ -package record +package record // import "bazil.org/fuse/fs/fstestutil/record" import ( "sync" diff --git a/vendor/src/bazil.org/fuse/fs/serve.go b/vendor/src/bazil.org/fuse/fs/serve.go index e5c3de865..a73a33b07 100644 --- a/vendor/src/bazil.org/fuse/fs/serve.go +++ b/vendor/src/bazil.org/fuse/fs/serve.go @@ -1,6 +1,6 @@ // FUSE service loop, for servers that wish to use it. -package fs +package fs // import "bazil.org/fuse/fs" import ( "encoding/binary" diff --git a/vendor/src/bazil.org/fuse/fuse.go b/vendor/src/bazil.org/fuse/fuse.go index c34d05981..424b2dce4 100644 --- a/vendor/src/bazil.org/fuse/fuse.go +++ b/vendor/src/bazil.org/fuse/fuse.go @@ -98,7 +98,7 @@ // Behavior and metadata of the mounted file system can be changed by // passing MountOption values to Mount. // -package fuse +package fuse // import "bazil.org/fuse" import ( "bytes" diff --git a/vendor/src/bazil.org/fuse/fuseutil/fuseutil.go b/vendor/src/bazil.org/fuse/fuseutil/fuseutil.go index c9e44de70..b3f52b73b 100644 --- a/vendor/src/bazil.org/fuse/fuseutil/fuseutil.go +++ b/vendor/src/bazil.org/fuse/fuseutil/fuseutil.go @@ -1,4 +1,4 @@ -package fuseutil +package fuseutil // import "bazil.org/fuse/fuseutil" import ( "bazil.org/fuse" diff --git a/vendor/src/bazil.org/fuse/syscallx/doc.go b/vendor/src/bazil.org/fuse/syscallx/doc.go index 8ceee43b0..c2c9b19c3 100644 --- a/vendor/src/bazil.org/fuse/syscallx/doc.go +++ b/vendor/src/bazil.org/fuse/syscallx/doc.go @@ -10,4 +10,4 @@ // // Options can be implemented with separate wrappers, in the style of // Linux getxattr/lgetxattr/fgetxattr. -package syscallx +package syscallx // import "bazil.org/fuse/syscallx" diff --git a/vendor/src/github.com/jessevdk/go-flags/.travis.yml b/vendor/src/github.com/jessevdk/go-flags/.travis.yml deleted file mode 100644 index 3165f0042..000000000 --- a/vendor/src/github.com/jessevdk/go-flags/.travis.yml +++ /dev/null @@ -1,35 +0,0 @@ -language: go - -install: - # go-flags - - go get -d -v ./... - - go build -v ./... - - # linting - - go get golang.org/x/tools/cmd/vet - - go get github.com/golang/lint - - go install github.com/golang/lint/golint - - # code coverage - - go get golang.org/x/tools/cmd/cover - - go get github.com/onsi/ginkgo/ginkgo - - go get github.com/modocache/gover - - if [ "$TRAVIS_SECURE_ENV_VARS" = "true" ]; then go get github.com/mattn/goveralls; fi - -script: - # go-flags - - $(exit $(gofmt -l . | wc -l)) - - go test -v ./... - - # linting - - go tool vet -all=true -v=true . || true - - $(go env GOPATH | awk 'BEGIN{FS=":"} {print $1}')/bin/golint ./... - - # code coverage - - $(go env GOPATH | awk 'BEGIN{FS=":"} {print $1}')/bin/ginkgo -r -cover - - $(go env GOPATH | awk 'BEGIN{FS=":"} {print $1}')/bin/gover - - if [ "$TRAVIS_SECURE_ENV_VARS" = "true" ]; then $(go env GOPATH | awk 'BEGIN{FS=":"} {print $1}')/bin/goveralls -coverprofile=gover.coverprofile -service=travis-ci -repotoken $COVERALLS_TOKEN; fi - -env: - # coveralls.io - secure: "RCYbiB4P0RjQRIoUx/vG/AjP3mmYCbzOmr86DCww1Z88yNcy3hYr3Cq8rpPtYU5v0g7wTpu4adaKIcqRE9xknYGbqj3YWZiCoBP1/n4Z+9sHW3Dsd9D/GRGeHUus0laJUGARjWoCTvoEtOgTdGQDoX7mH+pUUY0FBltNYUdOiiU=" diff --git a/vendor/src/github.com/juju/errors/.gitignore b/vendor/src/github.com/juju/errors/.gitignore deleted file mode 100644 index 836562412..000000000 --- a/vendor/src/github.com/juju/errors/.gitignore +++ /dev/null @@ -1,23 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe -*.test diff --git a/vendor/src/github.com/minio/minio-go/.gitignore b/vendor/src/github.com/minio/minio-go/.gitignore deleted file mode 100644 index acf19db3a..000000000 --- a/vendor/src/github.com/minio/minio-go/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -*~ -*.test \ No newline at end of file diff --git a/vendor/src/github.com/minio/minio-go/.travis.yml b/vendor/src/github.com/minio/minio-go/.travis.yml deleted file mode 100644 index 32873865b..000000000 --- a/vendor/src/github.com/minio/minio-go/.travis.yml +++ /dev/null @@ -1,23 +0,0 @@ -sudo: false -language: go - -os: -- linux -- osx - -env: -- ARCH=x86_64 -- ARCH=i686 - -go: -- 1.5.2 -- 1.5.3 - -script: -- diff -au <(gofmt -d .) <(printf "") -- go vet ./... -- go test -short -race -v ./... - -notifications: - slack: - secure: HrOX2k6F/sEl6Rr4m5vHOdJCIwV42be0kz1Jy/WSMvrl/fQ8YkldKviLeWh4aWt1kclsYhNQ4FqGML+RIZYsdOqej4fAw9Vi5pZkI1MzPJq0UjrtMqkqzvD90eDGQYCKwaXjEIN8cohwJeb6X0B0HKAd9sqJW5GH5SwnhH5WWP8= diff --git a/vendor/src/github.com/minio/minio-go/api_functional_v2_test.go b/vendor/src/github.com/minio/minio-go/api_functional_v2_test.go new file mode 100644 index 000000000..990e02810 --- /dev/null +++ b/vendor/src/github.com/minio/minio-go/api_functional_v2_test.go @@ -0,0 +1,1043 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio_test + +import ( + "bytes" + crand "crypto/rand" + "errors" + "io" + "io/ioutil" + "math/rand" + "net/http" + "os" + "testing" + "time" + + "github.com/minio/minio-go" +) + +// Tests bucket re-create errors. +func TestMakeBucketErrorV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket in 'eu-west-1'. + if err = c.MakeBucket(bucketName, "private", "eu-west-1"); err != nil { + t.Fatal("Error:", err, bucketName) + } + if err = c.MakeBucket(bucketName, "private", "eu-west-1"); err == nil { + t.Fatal("Error: make bucket should should fail for", bucketName) + } + // Verify valid error response from server. + if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" && + minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { + t.Fatal("Error: Invalid error returned by server", err) + } + if err = c.RemoveBucket(bucketName); err != nil { + t.Fatal("Error:", err, bucketName) + } +} + +// Test get object reader to not throw error on being closed twice. +func TestGetObjectClosedTwiceV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + buf := make([]byte, rand.Intn(1<<20)+32*1024) + + _, err = io.ReadFull(crand.Reader, buf) + if err != nil { + t.Fatal("Error:", err) + } + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano())) + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + if err := r.Close(); err != nil { + t.Fatal("Error:", err) + } + if err := r.Close(); err == nil { + t.Fatal("Error: object is already closed, should return error") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests removing partially uploaded objects. +func TestRemovePartiallyUploadedV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping function tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Enable tracing, write to stdout. + // c.TraceOn(os.Stderr) + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + reader, writer := io.Pipe() + go func() { + i := 0 + for i < 25 { + _, err = io.CopyN(writer, crand.Reader, 128*1024) + if err != nil { + t.Fatal("Error:", err, bucketName) + } + i++ + } + writer.CloseWithError(errors.New("Proactively closed to be verified later.")) + }() + + objectName := bucketName + "-resumable" + _, err = c.PutObject(bucketName, objectName, reader, "application/octet-stream") + if err == nil { + t.Fatal("Error: PutObject should fail.") + } + if err.Error() != "Proactively closed to be verified later." { + t.Fatal("Error:", err) + } + err = c.RemoveIncompleteUpload(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests resumable put object cloud to cloud. +func TestResumbalePutObjectV2(t *testing.T) { + // By passing 'go test -short' skips these tests. + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Enable tracing, write to stdout. + // c.TraceOn(os.Stderr) + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Create a temporary file. + file, err := ioutil.TempFile(os.TempDir(), "resumable") + if err != nil { + t.Fatal("Error:", err) + } + + // Copy 11MiB worth of random data. + n, err := io.CopyN(file, crand.Reader, 11*1024*1024) + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(11*1024*1024) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) + } + + // Close the file pro-actively for windows. + if err = file.Close(); err != nil { + t.Fatal("Error:", err) + } + + // New object name. + objectName := bucketName + "-resumable" + + // Upload the file. + n, err = c.FPutObject(bucketName, objectName, file.Name(), "application/octet-stream") + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(11*1024*1024) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) + } + + // Get the uploaded object. + reader, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err) + } + + // Upload now cloud to cloud. + n, err = c.PutObject(bucketName, objectName+"-put", reader, "application/octest-stream") + if err != nil { + t.Fatal("Error:", err) + } + + // Get object info. + objInfo, err := reader.Stat() + if err != nil { + t.Fatal("Error:", err) + } + if n != objInfo.Size { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", objInfo.Size, n) + } + + // Remove all temp files, objects and bucket. + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveObject(bucketName, objectName+"-put") + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + + err = os.Remove(file.Name()) + if err != nil { + t.Fatal("Error:", err) + } + +} + +// Tests resumable file based put object multipart upload. +func TestResumableFPutObjectV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Enable tracing, write to stdout. + // c.TraceOn(os.Stderr) + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + file, err := ioutil.TempFile(os.TempDir(), "resumable") + if err != nil { + t.Fatal("Error:", err) + } + + n, err := io.CopyN(file, crand.Reader, 11*1024*1024) + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(11*1024*1024) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) + } + + objectName := bucketName + "-resumable" + + n, err = c.FPutObject(bucketName, objectName, file.Name(), "application/octet-stream") + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(11*1024*1024) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) + } + + // Close the file pro-actively for windows. + file.Close() + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + + err = os.Remove(file.Name()) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests various bucket supported formats. +func TestMakeBucketRegionsV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket in 'eu-central-1'. + if err = c.MakeBucket(bucketName, "private", "eu-west-1"); err != nil { + t.Fatal("Error:", err, bucketName) + } + + if err = c.RemoveBucket(bucketName); err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Make a new bucket with '.' in its name, in 'us-west-2'. This + // request is internally staged into a path style instead of + // virtual host style. + if err = c.MakeBucket(bucketName+".withperiod", "private", "us-west-2"); err != nil { + t.Fatal("Error:", err, bucketName+".withperiod") + } + + // Remove the newly created bucket. + if err = c.RemoveBucket(bucketName + ".withperiod"); err != nil { + t.Fatal("Error:", err, bucketName+".withperiod") + } +} + +// Tests resumable put object multipart upload. +func TestResumablePutObjectV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // generate 11MB + buf := make([]byte, 11*1024*1024) + + _, err = io.ReadFull(crand.Reader, buf) + if err != nil { + t.Fatal("Error:", err) + } + + objectName := bucketName + "-resumable" + reader := bytes.NewReader(buf) + n, err := c.PutObject(bucketName, objectName, reader, "application/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests get object ReaderSeeker interface methods. +func TestGetObjectReadSeekFunctionalV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + buf := make([]byte, rand.Intn(1<<20)+32*1024) + + _, err = io.ReadFull(crand.Reader, buf) + if err != nil { + t.Fatal("Error:", err) + } + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano())) + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + + offset := int64(2048) + n, err = r.Seek(offset, 0) + if err != nil { + t.Fatal("Error:", err, offset) + } + if n != offset { + t.Fatalf("Error: number of bytes seeked does not match, want %v, got %v\n", + offset, n) + } + n, err = r.Seek(0, 1) + if err != nil { + t.Fatal("Error:", err) + } + if n != offset { + t.Fatalf("Error: number of current seek does not match, want %v, got %v\n", + offset, n) + } + _, err = r.Seek(offset, 2) + if err == nil { + t.Fatal("Error: seek on positive offset for whence '2' should error out") + } + n, err = r.Seek(-offset, 2) + if err != nil { + t.Fatal("Error:", err) + } + if n != 0 { + t.Fatalf("Error: number of bytes seeked back does not match, want 0, got %v\n", n) + } + var buffer bytes.Buffer + if _, err = io.CopyN(&buffer, r, st.Size); err != nil { + t.Fatal("Error:", err) + } + if !bytes.Equal(buf, buffer.Bytes()) { + t.Fatal("Error: Incorrect read bytes v/s original buffer.") + } + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests get object ReaderAt interface methods. +func TestGetObjectReadAtFunctionalV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.NewV2( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + buf := make([]byte, rand.Intn(1<<20)+32*1024) + + _, err = io.ReadFull(crand.Reader, buf) + if err != nil { + t.Fatal("Error:", err) + } + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano())) + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + + offset := int64(2048) + + // Read directly + buf2 := make([]byte, 512) + buf3 := make([]byte, 512) + buf4 := make([]byte, 512) + + m, err := r.ReadAt(buf2, offset) + if err != nil { + t.Fatal("Error:", err, st.Size, len(buf2), offset) + } + if m != len(buf2) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf2)) + } + if !bytes.Equal(buf2, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + offset += 512 + m, err = r.ReadAt(buf3, offset) + if err != nil { + t.Fatal("Error:", err, st.Size, len(buf3), offset) + } + if m != len(buf3) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf3)) + } + if !bytes.Equal(buf3, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + offset += 512 + m, err = r.ReadAt(buf4, offset) + if err != nil { + t.Fatal("Error:", err, st.Size, len(buf4), offset) + } + if m != len(buf4) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf4)) + } + if !bytes.Equal(buf4, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + + buf5 := make([]byte, n) + // Read the whole object. + m, err = r.ReadAt(buf5, 0) + if err != nil { + if err != io.EOF { + t.Fatal("Error:", err, len(buf5)) + } + } + if m != len(buf5) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf5)) + } + if !bytes.Equal(buf, buf5) { + t.Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") + } + + buf6 := make([]byte, n+1) + // Read the whole object and beyond. + _, err = r.ReadAt(buf6, 0) + if err != nil { + if err != io.EOF { + t.Fatal("Error:", err, len(buf6)) + } + } + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests comprehensive list of all methods. +func TestFunctionalV2(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + c, err := minio.NewV2( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable to debug + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate a random file name. + fileName := randString(60, rand.NewSource(time.Now().UnixNano())) + file, err := os.Create(fileName) + if err != nil { + t.Fatal("Error:", err) + } + for i := 0; i < 3; i++ { + buf := make([]byte, rand.Intn(1<<19)) + _, err = file.Write(buf) + if err != nil { + t.Fatal("Error:", err) + } + } + file.Close() + + // Verify if bucket exits and you have access. + err = c.BucketExists(bucketName) + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Make the bucket 'public read/write'. + err = c.SetBucketACL(bucketName, "public-read-write") + if err != nil { + t.Fatal("Error:", err) + } + + // Get the previously set acl. + acl, err := c.GetBucketACL(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + + // ACL must be 'public read/write'. + if acl != minio.BucketACL("public-read-write") { + t.Fatal("Error:", acl) + } + + // List all buckets. + buckets, err := c.ListBuckets() + if len(buckets) == 0 { + t.Fatal("Error: list buckets cannot be empty", buckets) + } + if err != nil { + t.Fatal("Error:", err) + } + + // Verify if previously created bucket is listed in list buckets. + bucketFound := false + for _, bucket := range buckets { + if bucket.Name == bucketName { + bucketFound = true + } + } + + // If bucket not found error out. + if !bucketFound { + t.Fatal("Error: bucket ", bucketName, "not found") + } + + objectName := bucketName + "unique" + + // Generate data + buf := make([]byte, rand.Intn(1<<19)) + _, err = io.ReadFull(crand.Reader, buf) + if err != nil { + t.Fatal("Error: ", err) + } + + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "") + if err != nil { + t.Fatal("Error: ", err) + } + if n != int64(len(buf)) { + t.Fatal("Error: bad length ", n, len(buf)) + } + + n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName+"-nolength") + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Instantiate a done channel to close all listing. + doneCh := make(chan struct{}) + defer close(doneCh) + + objFound := false + isRecursive := true // Recursive is true. + for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { + if obj.Key == objectName { + objFound = true + break + } + } + if !objFound { + t.Fatal("Error: object " + objectName + " not found.") + } + + incompObjNotFound := true + for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { + if objIncompl.Key != "" { + incompObjNotFound = false + break + } + } + if !incompObjNotFound { + t.Fatal("Error: unexpected dangling incomplete upload found.") + } + + newReader, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + + newReadBytes, err := ioutil.ReadAll(newReader) + if err != nil { + t.Fatal("Error: ", err) + } + + if !bytes.Equal(newReadBytes, buf) { + t.Fatal("Error: bytes mismatch.") + } + + err = c.FGetObject(bucketName, objectName, fileName+"-f") + if err != nil { + t.Fatal("Error: ", err) + } + + presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second) + if err != nil { + t.Fatal("Error: ", err) + } + + resp, err := http.Get(presignedGetURL) + if err != nil { + t.Fatal("Error: ", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatal("Error: ", resp.Status) + } + newPresignedBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal("Error: ", err) + } + if !bytes.Equal(newPresignedBytes, buf) { + t.Fatal("Error: bytes mismatch.") + } + + presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) + if err != nil { + t.Fatal("Error: ", err) + } + buf = make([]byte, rand.Intn(1<<20)) + _, err = io.ReadFull(crand.Reader, buf) + if err != nil { + t.Fatal("Error: ", err) + } + req, err := http.NewRequest("PUT", presignedPutURL, bytes.NewReader(buf)) + if err != nil { + t.Fatal("Error: ", err) + } + httpClient := &http.Client{} + resp, err = httpClient.Do(req) + if err != nil { + t.Fatal("Error: ", err) + } + + newReader, err = c.GetObject(bucketName, objectName+"-presigned") + if err != nil { + t.Fatal("Error: ", err) + } + + newReadBytes, err = ioutil.ReadAll(newReader) + if err != nil { + t.Fatal("Error: ", err) + } + + if !bytes.Equal(newReadBytes, buf) { + t.Fatal("Error: bytes mismatch.") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-f") + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-nolength") + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-presigned") + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + err = c.RemoveBucket(bucketName) + if err == nil { + t.Fatal("Error:") + } + if err.Error() != "The specified bucket does not exist" { + t.Fatal("Error: ", err) + } + if err = os.Remove(fileName); err != nil { + t.Fatal("Error: ", err) + } + if err = os.Remove(fileName + "-f"); err != nil { + t.Fatal("Error: ", err) + } +} diff --git a/vendor/src/github.com/minio/minio-go/api_functional_v4_test.go b/vendor/src/github.com/minio/minio-go/api_functional_v4_test.go new file mode 100644 index 000000000..5e88c6124 --- /dev/null +++ b/vendor/src/github.com/minio/minio-go/api_functional_v4_test.go @@ -0,0 +1,1072 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio_test + +import ( + "bytes" + crand "crypto/rand" + "errors" + "io" + "io/ioutil" + "math/rand" + "net/http" + "os" + "testing" + "time" + + "github.com/minio/minio-go" +) + +const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" +const ( + letterIdxBits = 6 // 6 bits to represent a letter index + letterIdxMask = 1<= 0; { + if remain == 0 { + cache, remain = src.Int63(), letterIdxMax + } + if idx := int(cache & letterIdxMask); idx < len(letterBytes) { + b[i] = letterBytes[idx] + i-- + } + cache >>= letterIdxBits + remain-- + } + return string(b[0:30]) +} + +// Tests bucket re-create errors. +func TestMakeBucketError(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket in 'eu-central-1'. + if err = c.MakeBucket(bucketName, "private", "eu-central-1"); err != nil { + t.Fatal("Error:", err, bucketName) + } + if err = c.MakeBucket(bucketName, "private", "eu-central-1"); err == nil { + t.Fatal("Error: make bucket should should fail for", bucketName) + } + // Verify valid error response from server. + if minio.ToErrorResponse(err).Code != "BucketAlreadyExists" && + minio.ToErrorResponse(err).Code != "BucketAlreadyOwnedByYou" { + t.Fatal("Error: Invalid error returned by server", err) + } + if err = c.RemoveBucket(bucketName); err != nil { + t.Fatal("Error:", err, bucketName) + } +} + +// Tests various bucket supported formats. +func TestMakeBucketRegions(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket in 'eu-central-1'. + if err = c.MakeBucket(bucketName, "private", "eu-central-1"); err != nil { + t.Fatal("Error:", err, bucketName) + } + + if err = c.RemoveBucket(bucketName); err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Make a new bucket with '.' in its name, in 'us-west-2'. This + // request is internally staged into a path style instead of + // virtual host style. + if err = c.MakeBucket(bucketName+".withperiod", "private", "us-west-2"); err != nil { + t.Fatal("Error:", err, bucketName+".withperiod") + } + + // Remove the newly created bucket. + if err = c.RemoveBucket(bucketName + ".withperiod"); err != nil { + t.Fatal("Error:", err, bucketName+".withperiod") + } +} + +// Test get object reader to not throw error on being closed twice. +func TestGetObjectClosedTwice(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + buf := make([]byte, rand.Intn(1<<20)+32*1024) + + _, err = io.ReadFull(crand.Reader, buf) + if err != nil { + t.Fatal("Error:", err) + } + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano())) + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + if err := r.Close(); err != nil { + t.Fatal("Error:", err) + } + if err := r.Close(); err == nil { + t.Fatal("Error: object is already closed, should return error") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests removing partially uploaded objects. +func TestRemovePartiallyUploaded(t *testing.T) { + if testing.Short() { + t.Skip("skipping function tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Enable tracing, write to stdout. + // c.TraceOn(os.Stderr) + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + reader, writer := io.Pipe() + go func() { + i := 0 + for i < 25 { + _, err = io.CopyN(writer, crand.Reader, 128*1024) + if err != nil { + t.Fatal("Error:", err, bucketName) + } + i++ + } + err := writer.CloseWithError(errors.New("Proactively closed to be verified later.")) + if err != nil { + t.Fatal("Error:", err) + } + }() + + objectName := bucketName + "-resumable" + _, err = c.PutObject(bucketName, objectName, reader, "application/octet-stream") + if err == nil { + t.Fatal("Error: PutObject should fail.") + } + if err.Error() != "Proactively closed to be verified later." { + t.Fatal("Error:", err) + } + err = c.RemoveIncompleteUpload(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests resumable put object cloud to cloud. +func TestResumbalePutObject(t *testing.T) { + // By passing 'go test -short' skips these tests. + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Enable tracing, write to stdout. + // c.TraceOn(os.Stderr) + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Create a temporary file. + file, err := ioutil.TempFile(os.TempDir(), "resumable") + if err != nil { + t.Fatal("Error:", err) + } + + // Copy 11MiB worth of random data. + n, err := io.CopyN(file, crand.Reader, 11*1024*1024) + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(11*1024*1024) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) + } + + // Close the file pro-actively for windows. + if err = file.Close(); err != nil { + t.Fatal("Error:", err) + } + + // New object name. + objectName := bucketName + "-resumable" + + // Upload the file. + n, err = c.FPutObject(bucketName, objectName, file.Name(), "application/octet-stream") + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(11*1024*1024) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) + } + + // Get the uploaded object. + reader, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err) + } + + // Upload now cloud to cloud. + n, err = c.PutObject(bucketName, objectName+"-put", reader, "application/octest-stream") + if err != nil { + t.Fatal("Error:", err) + } + + // Get object info. + objInfo, err := reader.Stat() + if err != nil { + t.Fatal("Error:", err) + } + if n != objInfo.Size { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", objInfo.Size, n) + } + + // Remove all temp files, objects and bucket. + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveObject(bucketName, objectName+"-put") + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + + err = os.Remove(file.Name()) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests resumable file based put object multipart upload. +func TestResumableFPutObject(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Enable tracing, write to stdout. + // c.TraceOn(os.Stderr) + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + file, err := ioutil.TempFile(os.TempDir(), "resumable") + if err != nil { + t.Fatal("Error:", err) + } + + n, err := io.CopyN(file, crand.Reader, 11*1024*1024) + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(11*1024*1024) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) + } + + // Close the file pro-actively for windows. + err = file.Close() + if err != nil { + t.Fatal("Error:", err) + } + + objectName := bucketName + "-resumable" + + n, err = c.FPutObject(bucketName, objectName, file.Name(), "application/octet-stream") + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(11*1024*1024) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", 11*1024*1024, n) + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + + err = os.Remove(file.Name()) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests resumable put object multipart upload. +func TestResumablePutObject(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate 11MB + buf := make([]byte, 11*1024*1024) + + _, err = io.ReadFull(crand.Reader, buf) + if err != nil { + t.Fatal("Error:", err) + } + + objectName := bucketName + "-resumable" + reader := bytes.NewReader(buf) + n, err := c.PutObject(bucketName, objectName, reader, "application/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests get object ReaderSeeker interface methods. +func TestGetObjectReadSeekFunctional(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + buf := make([]byte, rand.Intn(1<<20)+32*1024) + + _, err = io.ReadFull(crand.Reader, buf) + if err != nil { + t.Fatal("Error:", err) + } + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano())) + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + + offset := int64(2048) + n, err = r.Seek(offset, 0) + if err != nil { + t.Fatal("Error:", err, offset) + } + if n != offset { + t.Fatalf("Error: number of bytes seeked does not match, want %v, got %v\n", + offset, n) + } + n, err = r.Seek(0, 1) + if err != nil { + t.Fatal("Error:", err) + } + if n != offset { + t.Fatalf("Error: number of current seek does not match, want %v, got %v\n", + offset, n) + } + _, err = r.Seek(offset, 2) + if err == nil { + t.Fatal("Error: seek on positive offset for whence '2' should error out") + } + n, err = r.Seek(-offset, 2) + if err != nil { + t.Fatal("Error:", err) + } + if n != 0 { + t.Fatalf("Error: number of bytes seeked back does not match, want 0, got %v\n", n) + } + var buffer bytes.Buffer + if _, err = io.CopyN(&buffer, r, st.Size); err != nil { + t.Fatal("Error:", err) + } + if !bytes.Equal(buf, buffer.Bytes()) { + t.Fatal("Error: Incorrect read bytes v/s original buffer.") + } + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests get object ReaderAt interface methods. +func TestGetObjectReadAtFunctional(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + buf := make([]byte, rand.Intn(1<<20)+32*1024) + + _, err = io.ReadFull(crand.Reader, buf) + if err != nil { + t.Fatal("Error:", err) + } + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano())) + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // read the data back + r, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + + offset := int64(2048) + + // read directly + buf2 := make([]byte, 512) + buf3 := make([]byte, 512) + buf4 := make([]byte, 512) + + m, err := r.ReadAt(buf2, offset) + if err != nil { + t.Fatal("Error:", err, st.Size, len(buf2), offset) + } + if m != len(buf2) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf2)) + } + if !bytes.Equal(buf2, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + offset += 512 + m, err = r.ReadAt(buf3, offset) + if err != nil { + t.Fatal("Error:", err, st.Size, len(buf3), offset) + } + if m != len(buf3) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf3)) + } + if !bytes.Equal(buf3, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + offset += 512 + m, err = r.ReadAt(buf4, offset) + if err != nil { + t.Fatal("Error:", err, st.Size, len(buf4), offset) + } + if m != len(buf4) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf4)) + } + if !bytes.Equal(buf4, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two ReadAt from same offset.") + } + + buf5 := make([]byte, n) + // Read the whole object. + m, err = r.ReadAt(buf5, 0) + if err != nil { + if err != io.EOF { + t.Fatal("Error:", err, len(buf5)) + } + } + if m != len(buf5) { + t.Fatalf("Error: ReadAt read shorter bytes before reaching EOF, want %v, got %v\n", m, len(buf5)) + } + if !bytes.Equal(buf, buf5) { + t.Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") + } + + buf6 := make([]byte, n+1) + // Read the whole object and beyond. + _, err = r.ReadAt(buf6, 0) + if err != nil { + if err != io.EOF { + t.Fatal("Error:", err, len(buf6)) + } + } + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests comprehensive list of all methods. +func TestFunctional(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + c, err := minio.New( + "s3.amazonaws.com", + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + false, + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable to debug + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano())) + + // Make a new bucket. + err = c.MakeBucket(bucketName, "private", "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate a random file name. + fileName := randString(60, rand.NewSource(time.Now().UnixNano())) + file, err := os.Create(fileName) + if err != nil { + t.Fatal("Error:", err) + } + for i := 0; i < 3; i++ { + buf := make([]byte, rand.Intn(1<<19)) + _, err = file.Write(buf) + if err != nil { + t.Fatal("Error:", err) + } + } + file.Close() + + // Verify if bucket exits and you have access. + err = c.BucketExists(bucketName) + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Make the bucket 'public read/write'. + err = c.SetBucketACL(bucketName, "public-read-write") + if err != nil { + t.Fatal("Error:", err) + } + + // Get the previously set acl. + acl, err := c.GetBucketACL(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + + // ACL must be 'public read/write'. + if acl != minio.BucketACL("public-read-write") { + t.Fatal("Error:", acl) + } + + // List all buckets. + buckets, err := c.ListBuckets() + if len(buckets) == 0 { + t.Fatal("Error: list buckets cannot be empty", buckets) + } + if err != nil { + t.Fatal("Error:", err) + } + + // Verify if previously created bucket is listed in list buckets. + bucketFound := false + for _, bucket := range buckets { + if bucket.Name == bucketName { + bucketFound = true + } + } + + // If bucket not found error out. + if !bucketFound { + t.Fatal("Error: bucket ", bucketName, "not found") + } + + objectName := bucketName + "unique" + + // Generate data + buf := make([]byte, rand.Intn(1<<19)) + _, err = io.ReadFull(crand.Reader, buf) + if err != nil { + t.Fatal("Error: ", err) + } + + n, err := c.PutObject(bucketName, objectName, bytes.NewReader(buf), "") + if err != nil { + t.Fatal("Error: ", err) + } + if n != int64(len(buf)) { + t.Fatal("Error: bad length ", n, len(buf)) + } + + n, err = c.PutObject(bucketName, objectName+"-nolength", bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName+"-nolength") + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + // Instantiate a done channel to close all listing. + doneCh := make(chan struct{}) + defer close(doneCh) + + objFound := false + isRecursive := true // Recursive is true. + for obj := range c.ListObjects(bucketName, objectName, isRecursive, doneCh) { + if obj.Key == objectName { + objFound = true + break + } + } + if !objFound { + t.Fatal("Error: object " + objectName + " not found.") + } + + incompObjNotFound := true + for objIncompl := range c.ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh) { + if objIncompl.Key != "" { + incompObjNotFound = false + break + } + } + if !incompObjNotFound { + t.Fatal("Error: unexpected dangling incomplete upload found.") + } + + newReader, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + + newReadBytes, err := ioutil.ReadAll(newReader) + if err != nil { + t.Fatal("Error: ", err) + } + + if !bytes.Equal(newReadBytes, buf) { + t.Fatal("Error: bytes mismatch.") + } + + err = c.FGetObject(bucketName, objectName, fileName+"-f") + if err != nil { + t.Fatal("Error: ", err) + } + + presignedGetURL, err := c.PresignedGetObject(bucketName, objectName, 3600*time.Second) + if err != nil { + t.Fatal("Error: ", err) + } + + resp, err := http.Get(presignedGetURL) + if err != nil { + t.Fatal("Error: ", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatal("Error: ", resp.Status) + } + newPresignedBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Fatal("Error: ", err) + } + if !bytes.Equal(newPresignedBytes, buf) { + t.Fatal("Error: bytes mismatch.") + } + + presignedPutURL, err := c.PresignedPutObject(bucketName, objectName+"-presigned", 3600*time.Second) + if err != nil { + t.Fatal("Error: ", err) + } + buf = make([]byte, rand.Intn(1<<20)) + _, err = io.ReadFull(crand.Reader, buf) + if err != nil { + t.Fatal("Error: ", err) + } + req, err := http.NewRequest("PUT", presignedPutURL, bytes.NewReader(buf)) + if err != nil { + t.Fatal("Error: ", err) + } + httpClient := &http.Client{} + resp, err = httpClient.Do(req) + if err != nil { + t.Fatal("Error: ", err) + } + + newReader, err = c.GetObject(bucketName, objectName+"-presigned") + if err != nil { + t.Fatal("Error: ", err) + } + + newReadBytes, err = ioutil.ReadAll(newReader) + if err != nil { + t.Fatal("Error: ", err) + } + + if !bytes.Equal(newReadBytes, buf) { + t.Fatal("Error: bytes mismatch.") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-f") + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-nolength") + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveObject(bucketName, objectName+"-presigned") + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + err = c.RemoveBucket(bucketName) + if err == nil { + t.Fatal("Error:") + } + if err.Error() != "The specified bucket does not exist" { + t.Fatal("Error: ", err) + } + if err = os.Remove(fileName); err != nil { + t.Fatal("Error: ", err) + } + if err = os.Remove(fileName + "-f"); err != nil { + t.Fatal("Error: ", err) + } +} diff --git a/vendor/src/github.com/minio/minio-go/api_unit_test.go b/vendor/src/github.com/minio/minio-go/api_unit_test.go new file mode 100644 index 000000000..2afc666d8 --- /dev/null +++ b/vendor/src/github.com/minio/minio-go/api_unit_test.go @@ -0,0 +1,583 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "os" + "strings" + "testing" +) + +type customReader struct{} + +func (c *customReader) Read(p []byte) (n int, err error) { + return 0, nil +} + +func (c *customReader) Size() (n int64) { + return 10 +} + +// Tests getReaderSize() for various Reader types. +func TestGetReaderSize(t *testing.T) { + var reader io.Reader + size, err := getReaderSize(reader) + if err != nil { + t.Fatal("Error:", err) + } + if size != -1 { + t.Fatal("Reader shouldn't have any length.") + } + + bytesReader := bytes.NewReader([]byte("Hello World")) + size, err = getReaderSize(bytesReader) + if err != nil { + t.Fatal("Error:", err) + } + if size != int64(len("Hello World")) { + t.Fatalf("Reader length doesn't match got: %v, want: %v", size, len("Hello World")) + } + + size, err = getReaderSize(new(customReader)) + if err != nil { + t.Fatal("Error:", err) + } + if size != int64(10) { + t.Fatalf("Reader length doesn't match got: %v, want: %v", size, 10) + } + + stringsReader := strings.NewReader("Hello World") + size, err = getReaderSize(stringsReader) + if err != nil { + t.Fatal("Error:", err) + } + if size != int64(len("Hello World")) { + t.Fatalf("Reader length doesn't match got: %v, want: %v", size, len("Hello World")) + } + + // Create request channel. + reqCh := make(chan readRequest) + // Create response channel. + resCh := make(chan readResponse) + // Create done channel. + doneCh := make(chan struct{}) + // objectInfo. + objectInfo := ObjectInfo{Size: 10} + objectReader := newObject(reqCh, resCh, doneCh, objectInfo) + defer objectReader.Close() + + size, err = getReaderSize(objectReader) + if err != nil { + t.Fatal("Error:", err) + } + if size != int64(10) { + t.Fatalf("Reader length doesn't match got: %v, want: %v", size, 10) + } + + fileReader, err := ioutil.TempFile(os.TempDir(), "prefix") + if err != nil { + t.Fatal("Error:", err) + } + defer fileReader.Close() + defer os.RemoveAll(fileReader.Name()) + + size, err = getReaderSize(fileReader) + if err != nil { + t.Fatal("Error:", err) + } + if size == -1 { + t.Fatal("Reader length for file cannot be -1.") + } + + // Verify for standard input, output and error file descriptors. + size, err = getReaderSize(os.Stdin) + if err != nil { + t.Fatal("Error:", err) + } + if size != -1 { + t.Fatal("Stdin should have length of -1.") + } + size, err = getReaderSize(os.Stdout) + if err != nil { + t.Fatal("Error:", err) + } + if size != -1 { + t.Fatal("Stdout should have length of -1.") + } + size, err = getReaderSize(os.Stderr) + if err != nil { + t.Fatal("Error:", err) + } + if size != -1 { + t.Fatal("Stderr should have length of -1.") + } + file, err := os.Open(os.TempDir()) + if err != nil { + t.Fatal("Error:", err) + } + defer file.Close() + _, err = getReaderSize(file) + if err == nil { + t.Fatal("Input file as directory should throw an error.") + } +} + +// Tests valid hosts for location. +func TestValidBucketLocation(t *testing.T) { + s3Hosts := []struct { + bucketLocation string + endpoint string + }{ + {"us-east-1", "s3.amazonaws.com"}, + {"unknown", "s3.amazonaws.com"}, + {"ap-southeast-1", "s3-ap-southeast-1.amazonaws.com"}, + } + for _, s3Host := range s3Hosts { + endpoint := getS3Endpoint(s3Host.bucketLocation) + if endpoint != s3Host.endpoint { + t.Fatal("Error: invalid bucket location", endpoint) + } + } +} + +// Tests valid bucket names. +func TestBucketNames(t *testing.T) { + buckets := []struct { + name string + valid error + }{ + {".mybucket", ErrInvalidBucketName("Bucket name cannot start or end with a '.' dot.")}, + {"mybucket.", ErrInvalidBucketName("Bucket name cannot start or end with a '.' dot.")}, + {"mybucket-", ErrInvalidBucketName("Bucket name contains invalid characters.")}, + {"my", ErrInvalidBucketName("Bucket name cannot be smaller than 3 characters.")}, + {"", ErrInvalidBucketName("Bucket name cannot be empty.")}, + {"my..bucket", ErrInvalidBucketName("Bucket name cannot have successive periods.")}, + {"my.bucket.com", nil}, + {"my-bucket", nil}, + {"123my-bucket", nil}, + } + + for _, b := range buckets { + err := isValidBucketName(b.name) + if err != b.valid { + t.Fatal("Error:", err) + } + } +} + +// Tests temp file. +func TestTempFile(t *testing.T) { + tmpFile, err := newTempFile("testing") + if err != nil { + t.Fatal("Error:", err) + } + fileName := tmpFile.Name() + // Closing temporary file purges the file. + err = tmpFile.Close() + if err != nil { + t.Fatal("Error:", err) + } + st, err := os.Stat(fileName) + if err != nil && !os.IsNotExist(err) { + t.Fatal("Error:", err) + } + if err == nil && st != nil { + t.Fatal("Error: file should be deleted and should not exist.") + } +} + +// Tests url encoding. +func TestEncodeURL2Path(t *testing.T) { + type urlStrings struct { + objName string + encodedObjName string + } + + bucketName := "bucketName" + want := []urlStrings{ + { + objName: "本語", + encodedObjName: "%E6%9C%AC%E8%AA%9E", + }, + { + objName: "本語.1", + encodedObjName: "%E6%9C%AC%E8%AA%9E.1", + }, + { + objName: ">123>3123123", + encodedObjName: "%3E123%3E3123123", + }, + { + objName: "test 1 2.txt", + encodedObjName: "test%201%202.txt", + }, + { + objName: "test++ 1.txt", + encodedObjName: "test%2B%2B%201.txt", + }, + } + + for _, o := range want { + u, err := url.Parse(fmt.Sprintf("https://%s.s3.amazonaws.com/%s", bucketName, o.objName)) + if err != nil { + t.Fatal("Error:", err) + } + urlPath := "/" + bucketName + "/" + o.encodedObjName + if urlPath != encodeURL2Path(u) { + t.Fatal("Error") + } + } +} + +// Tests error response structure. +func TestErrorResponse(t *testing.T) { + var err error + err = ErrorResponse{ + Code: "Testing", + } + errResp := ToErrorResponse(err) + if errResp.Code != "Testing" { + t.Fatal("Type conversion failed, we have an empty struct.") + } + + // Test http response decoding. + var httpResponse *http.Response + // Set empty variables + httpResponse = nil + var bucketName, objectName string + + // Should fail with invalid argument. + err = httpRespToErrorResponse(httpResponse, bucketName, objectName) + errResp = ToErrorResponse(err) + if errResp.Code != "InvalidArgument" { + t.Fatal("Empty response input should return invalid argument.") + } +} + +// Tests signature calculation. +func TestSignatureCalculation(t *testing.T) { + req, err := http.NewRequest("GET", "https://s3.amazonaws.com", nil) + if err != nil { + t.Fatal("Error:", err) + } + req = signV4(*req, "", "", "us-east-1") + if req.Header.Get("Authorization") != "" { + t.Fatal("Error: anonymous credentials should not have Authorization header.") + } + + req = preSignV4(*req, "", "", "us-east-1", 0) + if strings.Contains(req.URL.RawQuery, "X-Amz-Signature") { + t.Fatal("Error: anonymous credentials should not have Signature query resource.") + } + + req = signV2(*req, "", "") + if req.Header.Get("Authorization") != "" { + t.Fatal("Error: anonymous credentials should not have Authorization header.") + } + + req = preSignV2(*req, "", "", 0) + if strings.Contains(req.URL.RawQuery, "Signature") { + t.Fatal("Error: anonymous credentials should not have Signature query resource.") + } + + req = signV4(*req, "ACCESS-KEY", "SECRET-KEY", "us-east-1") + if req.Header.Get("Authorization") == "" { + t.Fatal("Error: normal credentials should have Authorization header.") + } + + req = preSignV4(*req, "ACCESS-KEY", "SECRET-KEY", "us-east-1", 0) + if !strings.Contains(req.URL.RawQuery, "X-Amz-Signature") { + t.Fatal("Error: normal credentials should have Signature query resource.") + } + + req = signV2(*req, "ACCESS-KEY", "SECRET-KEY") + if req.Header.Get("Authorization") == "" { + t.Fatal("Error: normal credentials should have Authorization header.") + } + + req = preSignV2(*req, "ACCESS-KEY", "SECRET-KEY", 0) + if !strings.Contains(req.URL.RawQuery, "Signature") { + t.Fatal("Error: normal credentials should not have Signature query resource.") + } +} + +// Tests signature type. +func TestSignatureType(t *testing.T) { + clnt := Client{} + if !clnt.signature.isV4() { + t.Fatal("Error") + } + clnt.signature = SignatureV2 + if !clnt.signature.isV2() { + t.Fatal("Error") + } + if clnt.signature.isV4() { + t.Fatal("Error") + } + clnt.signature = SignatureV4 + if !clnt.signature.isV4() { + t.Fatal("Error") + } +} + +// Tests bucket acl types. +func TestBucketACLTypes(t *testing.T) { + want := map[string]bool{ + "private": true, + "public-read": true, + "public-read-write": true, + "authenticated-read": true, + "invalid": false, + } + for acl, ok := range want { + if BucketACL(acl).isValidBucketACL() != ok { + t.Fatal("Error") + } + } +} + +// Tests optimal part size. +func TestPartSize(t *testing.T) { + totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(5000000000000000000) + if err == nil { + t.Fatal("Error: should fail") + } + totalPartsCount, partSize, lastPartSize, err = optimalPartInfo(5497558138880) + if err != nil { + t.Fatal("Error: ", err) + } + if totalPartsCount != 9987 { + t.Fatalf("Error: expecting total parts count of 9987: got %v instead", totalPartsCount) + } + if partSize != 550502400 { + t.Fatalf("Error: expecting part size of 550502400: got %v instead", partSize) + } + if lastPartSize != 241172480 { + t.Fatalf("Error: expecting last part size of 241172480: got %v instead", lastPartSize) + } + totalPartsCount, partSize, lastPartSize, err = optimalPartInfo(5000000000) + if err != nil { + t.Fatal("Error:", err) + } + if partSize != minPartSize { + t.Fatalf("Error: expecting part size of %v: got %v instead", minPartSize, partSize) + } + totalPartsCount, partSize, lastPartSize, err = optimalPartInfo(-1) + if err != nil { + t.Fatal("Error:", err) + } + if totalPartsCount != 9987 { + t.Fatalf("Error: expecting total parts count of 9987: got %v instead", totalPartsCount) + } + if partSize != 550502400 { + t.Fatalf("Error: expecting part size of 550502400: got %v instead", partSize) + } + if lastPartSize != 241172480 { + t.Fatalf("Error: expecting last part size of 241172480: got %v instead", lastPartSize) + } +} + +// Tests url encoding. +func TestURLEncoding(t *testing.T) { + type urlStrings struct { + name string + encodedName string + } + + want := []urlStrings{ + { + name: "bigfile-1._%", + encodedName: "bigfile-1._%25", + }, + { + name: "本語", + encodedName: "%E6%9C%AC%E8%AA%9E", + }, + { + name: "本語.1", + encodedName: "%E6%9C%AC%E8%AA%9E.1", + }, + { + name: ">123>3123123", + encodedName: "%3E123%3E3123123", + }, + { + name: "test 1 2.txt", + encodedName: "test%201%202.txt", + }, + { + name: "test++ 1.txt", + encodedName: "test%2B%2B%201.txt", + }, + } + + for _, u := range want { + if u.encodedName != urlEncodePath(u.name) { + t.Fatal("Error") + } + } +} + +// Tests constructing valid endpoint url. +func TestGetEndpointURL(t *testing.T) { + if _, err := getEndpointURL("s3.amazonaws.com", false); err != nil { + t.Fatal("Error:", err) + } + if _, err := getEndpointURL("192.168.1.1", false); err != nil { + t.Fatal("Error:", err) + } + if _, err := getEndpointURL("13333.123123.-", false); err == nil { + t.Fatal("Error") + } + if _, err := getEndpointURL("s3.aamzza.-", false); err == nil { + t.Fatal("Error") + } + if _, err := getEndpointURL("s3.amazonaws.com:443", false); err == nil { + t.Fatal("Error") + } +} + +// Tests valid ip address. +func TestValidIPAddr(t *testing.T) { + type validIP struct { + ip string + valid bool + } + + want := []validIP{ + { + ip: "192.168.1.1", + valid: true, + }, + { + ip: "192.1.8", + valid: false, + }, + { + ip: "..192.", + valid: false, + }, + { + ip: "192.168.1.1.1", + valid: false, + }, + } + for _, w := range want { + valid := isValidIP(w.ip) + if valid != w.valid { + t.Fatal("Error") + } + } +} + +// Tests valid endpoint domain. +func TestValidEndpointDomain(t *testing.T) { + type validEndpoint struct { + endpointDomain string + valid bool + } + + want := []validEndpoint{ + { + endpointDomain: "s3.amazonaws.com", + valid: true, + }, + { + endpointDomain: "s3.amazonaws.com_", + valid: false, + }, + { + endpointDomain: "%$$$", + valid: false, + }, + { + endpointDomain: "s3.amz.test.com", + valid: true, + }, + { + endpointDomain: "s3.%%", + valid: false, + }, + { + endpointDomain: "localhost", + valid: true, + }, + { + endpointDomain: "-localhost", + valid: false, + }, + { + endpointDomain: "", + valid: false, + }, + { + endpointDomain: "\n \t", + valid: false, + }, + { + endpointDomain: " ", + valid: false, + }, + } + for _, w := range want { + valid := isValidDomain(w.endpointDomain) + if valid != w.valid { + t.Fatal("Error:", w.endpointDomain) + } + } +} + +// Tests valid endpoint url. +func TestValidEndpointURL(t *testing.T) { + type validURL struct { + url string + valid bool + } + want := []validURL{ + { + url: "https://s3.amazonaws.com", + valid: true, + }, + { + url: "https://s3.amazonaws.com/bucket/object", + valid: false, + }, + { + url: "192.168.1.1", + valid: false, + }, + } + for _, w := range want { + u, err := url.Parse(w.url) + if err != nil { + t.Fatal("Error:", err) + } + valid := false + if err := isValidEndpointURL(u); err == nil { + valid = true + } + if valid != w.valid { + t.Fatal("Error") + } + } +} diff --git a/vendor/src/github.com/pkg/sftp/.gitignore b/vendor/src/github.com/pkg/sftp/.gitignore deleted file mode 100644 index 9fc1e3d24..000000000 --- a/vendor/src/github.com/pkg/sftp/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -.*.swo -.*.swp - -server_standalone/server_standalone - -examples/sftp-server/id_rsa -examples/sftp-server/id_rsa.pub -examples/sftp-server/sftp-server diff --git a/vendor/src/github.com/pkg/sftp/.travis.yml b/vendor/src/github.com/pkg/sftp/.travis.yml deleted file mode 100644 index 9ef70d869..000000000 --- a/vendor/src/github.com/pkg/sftp/.travis.yml +++ /dev/null @@ -1,21 +0,0 @@ -language: go -go_import_path: github.com/pkg/sftp -go: - - 1.5.2 - - 1.4.3 - - tip - -sudo: false - -addons: - ssh_known_hosts: - - bitbucket.org - -install: - - go get -t -v ./... - - ssh-keygen -t rsa -q -P "" -f /home/travis/.ssh/id_rsa - -script: - - go test -integration -v ./... - - go test -testserver -v ./... - - go test -integration -testserver -v ./... diff --git a/vendor/src/github.com/pkg/sftp/attrs_test.go b/vendor/src/github.com/pkg/sftp/attrs_test.go new file mode 100644 index 000000000..e234649cd --- /dev/null +++ b/vendor/src/github.com/pkg/sftp/attrs_test.go @@ -0,0 +1,45 @@ +package sftp + +import ( + "bytes" + "os" + "reflect" + "testing" + "time" +) + +// ensure that attrs implemenst os.FileInfo +var _ os.FileInfo = new(fileInfo) + +var unmarshalAttrsTests = []struct { + b []byte + want *fileInfo + rest []byte +}{ + {marshal(nil, struct{ Flags uint32 }{}), &fileInfo{mtime: time.Unix(int64(0), 0)}, nil}, + {marshal(nil, struct { + Flags uint32 + Size uint64 + }{ssh_FILEXFER_ATTR_SIZE, 20}), &fileInfo{size: 20, mtime: time.Unix(int64(0), 0)}, nil}, + {marshal(nil, struct { + Flags uint32 + Size uint64 + Permissions uint32 + }{ssh_FILEXFER_ATTR_SIZE | ssh_FILEXFER_ATTR_PERMISSIONS, 20, 0644}), &fileInfo{size: 20, mode: os.FileMode(0644), mtime: time.Unix(int64(0), 0)}, nil}, + {marshal(nil, struct { + Flags uint32 + Size uint64 + UID, GID, Permissions uint32 + }{ssh_FILEXFER_ATTR_SIZE | ssh_FILEXFER_ATTR_UIDGID | ssh_FILEXFER_ATTR_UIDGID | ssh_FILEXFER_ATTR_PERMISSIONS, 20, 1000, 1000, 0644}), &fileInfo{size: 20, mode: os.FileMode(0644), mtime: time.Unix(int64(0), 0)}, nil}, +} + +func TestUnmarshalAttrs(t *testing.T) { + for _, tt := range unmarshalAttrsTests { + stat, rest := unmarshalAttrs(tt.b) + got := fileInfoFromStat(stat, "") + tt.want.sys = got.Sys() + if !reflect.DeepEqual(got, tt.want) || !bytes.Equal(tt.rest, rest) { + t.Errorf("unmarshalAttrs(%#v): want %#v, %#v, got: %#v, %#v", tt.b, tt.want, tt.rest, got, rest) + } + } +} diff --git a/vendor/src/github.com/pkg/sftp/client_integration_darwin_test.go b/vendor/src/github.com/pkg/sftp/client_integration_darwin_test.go new file mode 100644 index 000000000..0f7cd01cd --- /dev/null +++ b/vendor/src/github.com/pkg/sftp/client_integration_darwin_test.go @@ -0,0 +1,46 @@ +package sftp + +import ( + "syscall" + "testing" +) + +const sftpServer = "/usr/libexec/sftp-server" + +func TestClientStatVFS(t *testing.T) { + if *testServerImpl { + t.Skipf("go server does not support FXP_EXTENDED") + } + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + vfs, err := sftp.StatVFS("/") + if err != nil { + t.Fatal(err) + } + + // get system stats + s := syscall.Statfs_t{} + err = syscall.Statfs("/", &s) + if err != nil { + t.Fatal(err) + } + + // check some stats + if vfs.Files != uint64(s.Files) { + t.Fatal("fr_size does not match") + } + + if vfs.Bfree != uint64(s.Bfree) { + t.Fatal("f_bsize does not match") + } + + if vfs.Favail != uint64(s.Ffree) { + t.Fatal("f_namemax does not match") + } + + if vfs.Bavail != s.Bavail { + t.Fatal("f_bavail does not match") + } +} diff --git a/vendor/src/github.com/pkg/sftp/client_integration_linux_test.go b/vendor/src/github.com/pkg/sftp/client_integration_linux_test.go new file mode 100644 index 000000000..1663ad42a --- /dev/null +++ b/vendor/src/github.com/pkg/sftp/client_integration_linux_test.go @@ -0,0 +1,46 @@ +package sftp + +import ( + "syscall" + "testing" +) + +const sftpServer = "/usr/lib/openssh/sftp-server" + +func TestClientStatVFS(t *testing.T) { + if *testServerImpl { + t.Skipf("go server does not support FXP_EXTENDED") + } + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + vfs, err := sftp.StatVFS("/") + if err != nil { + t.Fatal(err) + } + + // get system stats + s := syscall.Statfs_t{} + err = syscall.Statfs("/", &s) + if err != nil { + t.Fatal(err) + } + + // check some stats + if vfs.Frsize != uint64(s.Frsize) { + t.Fatal("fr_size does not match") + } + + if vfs.Bsize != uint64(s.Bsize) { + t.Fatal("f_bsize does not match") + } + + if vfs.Namemax != uint64(s.Namelen) { + t.Fatal("f_namemax does not match") + } + + if vfs.Bavail != s.Bavail { + t.Fatal("f_bavail does not match") + } +} diff --git a/vendor/src/github.com/pkg/sftp/client_integration_test.go b/vendor/src/github.com/pkg/sftp/client_integration_test.go new file mode 100644 index 000000000..8fa7f95a0 --- /dev/null +++ b/vendor/src/github.com/pkg/sftp/client_integration_test.go @@ -0,0 +1,1616 @@ +package sftp + +// sftp integration tests +// enable with -integration + +import ( + "crypto/sha1" + "flag" + "io" + "io/ioutil" + "math/rand" + "os" + "os/exec" + "os/user" + "path" + "path/filepath" + "reflect" + "regexp" + "strconv" + "testing" + "testing/quick" + "time" + + "github.com/kr/fs" +) + +const ( + READONLY = true + READWRITE = false + NO_DELAY time.Duration = 0 + + debuglevel = "ERROR" // set to "DEBUG" for debugging +) + +var testServerImpl = flag.Bool("testserver", false, "perform integration tests against sftp package server instance") +var testIntegration = flag.Bool("integration", false, "perform integration tests against sftp server process") +var testSftp = flag.String("sftp", sftpServer, "location of the sftp server binary") + +type delayedWrite struct { + t time.Time + b []byte +} + +// delayedWriter wraps a writer and artificially delays the write. This is +// meant to mimic connections with various latencies. Error's returned from the +// underlying writer will panic so this should only be used over reliable +// connections. +type delayedWriter struct { + w io.WriteCloser + ch chan delayedWrite + closed chan struct{} +} + +func newDelayedWriter(w io.WriteCloser, delay time.Duration) io.WriteCloser { + ch := make(chan delayedWrite, 128) + closed := make(chan struct{}) + go func() { + for writeMsg := range ch { + time.Sleep(writeMsg.t.Add(delay).Sub(time.Now())) + n, err := w.Write(writeMsg.b) + if err != nil { + panic("write error") + } + if n < len(writeMsg.b) { + panic("showrt write") + } + } + w.Close() + close(closed) + }() + return delayedWriter{w: w, ch: ch, closed: closed} +} + +func (w delayedWriter) Write(b []byte) (int, error) { + bcopy := make([]byte, len(b)) + copy(bcopy, b) + w.ch <- delayedWrite{t: time.Now(), b: bcopy} + return len(b), nil +} + +func (w delayedWriter) Close() error { + close(w.ch) + <-w.closed + return nil +} + +func testClientGoSvr(t testing.TB, readonly bool, delay time.Duration) (*Client, *exec.Cmd) { + txPipeRd, txPipeWr, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + rxPipeRd, rxPipeWr, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + + options := []ServerOption{WithDebug(os.Stderr)} + if readonly { + options = append(options, ReadOnly()) + } + + server, err := NewServer( + txPipeRd, + rxPipeWr, + options..., + ) + if err != nil { + t.Fatal(err) + } + go server.Serve() + + var ctx io.WriteCloser = txPipeWr + if delay > NO_DELAY { + ctx = newDelayedWriter(ctx, delay) + } + + client, err := NewClientPipe(rxPipeRd, ctx) + if err != nil { + t.Fatal(err) + } + + // dummy command... + return client, exec.Command("true") +} + +// testClient returns a *Client connected to a localy running sftp-server +// the *exec.Cmd returned must be defer Wait'd. +func testClient(t testing.TB, readonly bool, delay time.Duration) (*Client, *exec.Cmd) { + if !*testIntegration { + t.Skip("skipping intergration test") + } + + if *testServerImpl { + return testClientGoSvr(t, readonly, delay) + } + + cmd := exec.Command(*testSftp, "-e", "-R", "-l", debuglevel) // log to stderr, read only + if !readonly { + cmd = exec.Command(*testSftp, "-e", "-l", debuglevel) // log to stderr + } + cmd.Stderr = os.Stdout + pw, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + if delay > NO_DELAY { + pw = newDelayedWriter(pw, delay) + } + pr, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + if err := cmd.Start(); err != nil { + t.Skipf("could not start sftp-server process: %v", err) + } + + sftp, err := NewClientPipe(pr, pw) + if err != nil { + t.Fatal(err) + } + + return sftp, cmd +} + +func TestNewClient(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + + if err := sftp.Close(); err != nil { + t.Fatal(err) + } +} + +func TestClientLstat(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + + want, err := os.Lstat(f.Name()) + if err != nil { + t.Fatal(err) + } + + got, err := sftp.Lstat(f.Name()) + if err != nil { + t.Fatal(err) + } + + if !sameFile(want, got) { + t.Fatalf("Lstat(%q): want %#v, got %#v", f.Name(), want, got) + } +} + +func TestClientLstatIsNotExist(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + os.Remove(f.Name()) + + if _, err := sftp.Lstat(f.Name()); !os.IsNotExist(err) { + t.Errorf("os.IsNotExist(%v) = false, want true", err) + } +} + +func TestClientMkdir(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + dir, err := ioutil.TempDir("", "sftptest") + if err != nil { + t.Fatal(err) + } + sub := path.Join(dir, "mkdir1") + if err := sftp.Mkdir(sub); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(sub); err != nil { + t.Fatal(err) + } +} + +func TestClientOpen(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + + got, err := sftp.Open(f.Name()) + if err != nil { + t.Fatal(err) + } + if err := got.Close(); err != nil { + t.Fatal(err) + } +} + +func TestClientOpenIsNotExist(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + if _, err := sftp.Open("/doesnt/exist/"); !os.IsNotExist(err) { + t.Errorf("os.IsNotExist(%v) = false, want true", err) + } +} + +func TestClientStatIsNotExist(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + if _, err := sftp.Stat("/doesnt/exist/"); !os.IsNotExist(err) { + t.Errorf("os.IsNotExist(%v) = false, want true", err) + } +} + +const seekBytes = 128 * 1024 + +type seek struct { + offset int64 +} + +func (s seek) Generate(r *rand.Rand, _ int) reflect.Value { + s.offset = int64(r.Int31n(seekBytes)) + return reflect.ValueOf(s) +} + +func (s seek) set(t *testing.T, r io.ReadSeeker) { + if _, err := r.Seek(s.offset, os.SEEK_SET); err != nil { + t.Fatalf("error while seeking with %+v: %v", s, err) + } +} + +func (s seek) current(t *testing.T, r io.ReadSeeker) { + const mid = seekBytes / 2 + + skip := s.offset / 2 + if s.offset > mid { + skip = -skip + } + + if _, err := r.Seek(mid, os.SEEK_SET); err != nil { + t.Fatalf("error seeking to midpoint with %+v: %v", s, err) + } + if _, err := r.Seek(skip, os.SEEK_CUR); err != nil { + t.Fatalf("error seeking from %d with %+v: %v", mid, s, err) + } +} + +func (s seek) end(t *testing.T, r io.ReadSeeker) { + if _, err := r.Seek(-s.offset, os.SEEK_END); err != nil { + t.Fatalf("error seeking from end with %+v: %v", s, err) + } +} + +func TestClientSeek(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + fOS, err := ioutil.TempFile("", "seek-test") + if err != nil { + t.Fatal(err) + } + defer fOS.Close() + + fSFTP, err := sftp.Open(fOS.Name()) + if err != nil { + t.Fatal(err) + } + defer fSFTP.Close() + + writeN(t, fOS, seekBytes) + + if err := quick.CheckEqual( + func(s seek) (string, int64) { s.set(t, fOS); return readHash(t, fOS) }, + func(s seek) (string, int64) { s.set(t, fSFTP); return readHash(t, fSFTP) }, + nil, + ); err != nil { + t.Errorf("Seek: expected equal absolute seeks: %v", err) + } + + if err := quick.CheckEqual( + func(s seek) (string, int64) { s.current(t, fOS); return readHash(t, fOS) }, + func(s seek) (string, int64) { s.current(t, fSFTP); return readHash(t, fSFTP) }, + nil, + ); err != nil { + t.Errorf("Seek: expected equal seeks from middle: %v", err) + } + + if err := quick.CheckEqual( + func(s seek) (string, int64) { s.end(t, fOS); return readHash(t, fOS) }, + func(s seek) (string, int64) { s.end(t, fSFTP); return readHash(t, fSFTP) }, + nil, + ); err != nil { + t.Errorf("Seek: expected equal seeks from end: %v", err) + } +} + +func TestClientCreate(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + defer f.Close() + defer os.Remove(f.Name()) + + f2, err := sftp.Create(f.Name()) + if err != nil { + t.Fatal(err) + } + defer f2.Close() +} + +func TestClientAppend(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + defer f.Close() + defer os.Remove(f.Name()) + + f2, err := sftp.OpenFile(f.Name(), os.O_RDWR|os.O_APPEND) + if err != nil { + t.Fatal(err) + } + defer f2.Close() +} + +func TestClientCreateFailed(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + defer f.Close() + defer os.Remove(f.Name()) + + f2, err := sftp.Create(f.Name()) + if err1, ok := err.(*StatusError); !ok || err1.Code != ssh_FX_PERMISSION_DENIED { + t.Fatalf("Create: want: %v, got %#v", ssh_FX_PERMISSION_DENIED, err) + } + if err == nil { + f2.Close() + } +} + +func TestClientFileName(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + + f2, err := sftp.Open(f.Name()) + if err != nil { + t.Fatal(err) + } + + if got, want := f2.Name(), f.Name(); got != want { + t.Fatalf("Name: got %q want %q", want, got) + } +} + +func TestClientFileStat(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + + want, err := os.Lstat(f.Name()) + if err != nil { + t.Fatal(err) + } + + f2, err := sftp.Open(f.Name()) + if err != nil { + t.Fatal(err) + } + + got, err := f2.Stat() + if err != nil { + t.Fatal(err) + } + + if !sameFile(want, got) { + t.Fatalf("Lstat(%q): want %#v, got %#v", f.Name(), want, got) + } +} + +func TestClientRemove(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + if err := sftp.Remove(f.Name()); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(f.Name()); !os.IsNotExist(err) { + t.Fatal(err) + } +} + +func TestClientRemoveDir(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + dir, err := ioutil.TempDir("", "sftptest") + if err != nil { + t.Fatal(err) + } + if err := sftp.Remove(dir); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(dir); !os.IsNotExist(err) { + t.Fatal(err) + } +} + +func TestClientRemoveFailed(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + if err := sftp.Remove(f.Name()); err == nil { + t.Fatalf("Remove(%v): want: permission denied, got %v", f.Name(), err) + } + if _, err := os.Lstat(f.Name()); err != nil { + t.Fatal(err) + } +} + +func TestClientRename(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + f2 := f.Name() + ".new" + if err := sftp.Rename(f.Name(), f2); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(f.Name()); !os.IsNotExist(err) { + t.Fatal(err) + } + if _, err := os.Lstat(f2); err != nil { + t.Fatal(err) + } +} + +func TestClientGetwd(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + lwd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + rwd, err := sftp.Getwd() + if err != nil { + t.Fatal(err) + } + if !path.IsAbs(rwd) { + t.Fatalf("Getwd: wanted absolute path, got %q", rwd) + } + if lwd != rwd { + t.Fatalf("Getwd: want %q, got %q", lwd, rwd) + } +} + +func TestClientReadLink(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + f2 := f.Name() + ".sym" + if err := os.Symlink(f.Name(), f2); err != nil { + t.Fatal(err) + } + if rl, err := sftp.ReadLink(f2); err != nil { + t.Fatal(err) + } else if rl != f.Name() { + t.Fatalf("unexpected link target: %v, not %v", rl, f.Name()) + } +} + +func TestClientSymlink(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + f2 := f.Name() + ".sym" + if err := sftp.Symlink(f.Name(), f2); err != nil { + t.Fatal(err) + } + if rl, err := sftp.ReadLink(f2); err != nil { + t.Fatal(err) + } else if rl != f.Name() { + t.Fatalf("unexpected link target: %v, not %v", rl, f.Name()) + } +} + +func TestClientChmod(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + if err := sftp.Chmod(f.Name(), 0531); err != nil { + t.Fatal(err) + } + if stat, err := os.Stat(f.Name()); err != nil { + t.Fatal(err) + } else if stat.Mode()&os.ModePerm != 0531 { + t.Fatalf("invalid perm %o\n", stat.Mode()) + } +} + +func TestClientChmodReadonly(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + if err := sftp.Chmod(f.Name(), 0531); err == nil { + t.Fatal("expected error") + } +} + +func TestClientChown(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + usr, err := user.Current() + if err != nil { + t.Fatal(err) + } + chownto, err := user.Lookup("daemon") // seems common-ish... + if err != nil { + t.Fatal(err) + } + + if usr.Uid != "0" { + t.Log("must be root to run chown tests") + t.Skip() + } + toUID, err := strconv.Atoi(chownto.Uid) + if err != nil { + t.Fatal(err) + } + toGID, err := strconv.Atoi(chownto.Gid) + if err != nil { + t.Fatal(err) + } + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + before, err := exec.Command("ls", "-nl", f.Name()).Output() + if err != nil { + t.Fatal(err) + } + if err := sftp.Chown(f.Name(), toUID, toGID); err != nil { + t.Fatal(err) + } + after, err := exec.Command("ls", "-nl", f.Name()).Output() + if err != nil { + t.Fatal(err) + } + + spaceRegex := regexp.MustCompile(`\s+`) + + beforeWords := spaceRegex.Split(string(before), -1) + if beforeWords[2] != "0" { + t.Fatalf("bad previous user? should be root") + } + afterWords := spaceRegex.Split(string(after), -1) + if afterWords[2] != chownto.Uid || afterWords[3] != chownto.Gid { + t.Fatalf("bad chown: %#v", afterWords) + } + t.Logf("before: %v", string(before)) + t.Logf(" after: %v", string(after)) +} + +func TestClientChownReadonly(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + usr, err := user.Current() + if err != nil { + t.Fatal(err) + } + chownto, err := user.Lookup("daemon") // seems common-ish... + if err != nil { + t.Fatal(err) + } + + if usr.Uid != "0" { + t.Log("must be root to run chown tests") + t.Skip() + } + toUID, err := strconv.Atoi(chownto.Uid) + if err != nil { + t.Fatal(err) + } + toGID, err := strconv.Atoi(chownto.Gid) + if err != nil { + t.Fatal(err) + } + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + if err := sftp.Chown(f.Name(), toUID, toGID); err == nil { + t.Fatal("expected error") + } +} + +func TestClientChtimes(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + + atime := time.Date(2013, 2, 23, 13, 24, 35, 0, time.UTC) + mtime := time.Date(1985, 6, 12, 6, 6, 6, 0, time.UTC) + if err := sftp.Chtimes(f.Name(), atime, mtime); err != nil { + t.Fatal(err) + } + if stat, err := os.Stat(f.Name()); err != nil { + t.Fatal(err) + } else if stat.ModTime().Sub(mtime) != 0 { + t.Fatalf("incorrect mtime: %v vs %v", stat.ModTime(), mtime) + } +} + +func TestClientChtimesReadonly(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + + atime := time.Date(2013, 2, 23, 13, 24, 35, 0, time.UTC) + mtime := time.Date(1985, 6, 12, 6, 6, 6, 0, time.UTC) + if err := sftp.Chtimes(f.Name(), atime, mtime); err == nil { + t.Fatal("expected error") + } +} + +func TestClientTruncate(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + fname := f.Name() + + if n, err := f.Write([]byte("hello world")); n != 11 || err != nil { + t.Fatal(err) + } + f.Close() + + if err := sftp.Truncate(fname, 5); err != nil { + t.Fatal(err) + } + if stat, err := os.Stat(fname); err != nil { + t.Fatal(err) + } else if stat.Size() != 5 { + t.Fatalf("unexpected size: %d", stat.Size()) + } +} + +func TestClientTruncateReadonly(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + t.Fatal(err) + } + fname := f.Name() + + if n, err := f.Write([]byte("hello world")); n != 11 || err != nil { + t.Fatal(err) + } + f.Close() + + if err := sftp.Truncate(fname, 5); err == nil { + t.Fatal("expected error") + } + if stat, err := os.Stat(fname); err != nil { + t.Fatal(err) + } else if stat.Size() != 11 { + t.Fatalf("unexpected size: %d", stat.Size()) + } +} + +func sameFile(want, got os.FileInfo) bool { + return want.Name() == got.Name() && + want.Size() == got.Size() +} + +func TestClientReadSimple(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + d, err := ioutil.TempDir("", "sftptest") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(d) + + f, err := ioutil.TempFile(d, "read-test") + if err != nil { + t.Fatal(err) + } + fname := f.Name() + f.Write([]byte("hello")) + f.Close() + + f2, err := sftp.Open(fname) + if err != nil { + t.Fatal(err) + } + defer f2.Close() + stuff := make([]byte, 32) + n, err := f2.Read(stuff) + if err != nil && err != io.EOF { + t.Fatalf("err: %v", err) + } + if n != 5 { + t.Fatalf("n should be 5, is %v", n) + } + if string(stuff[0:5]) != "hello" { + t.Fatalf("invalid contents") + } +} + +func TestClientReadDir(t *testing.T) { + sftp1, cmd1 := testClient(t, READONLY, NO_DELAY) + sftp2, cmd2 := testClientGoSvr(t, READONLY, NO_DELAY) + defer cmd1.Wait() + defer cmd2.Wait() + defer sftp1.Close() + defer sftp2.Close() + + dir := "/dev/" + + d, err := os.Open(dir) + if err != nil { + t.Fatal(err) + } + defer d.Close() + osfiles, err := d.Readdir(4096) + if err != nil { + t.Fatal(err) + } + + sftp1Files, err := sftp1.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + sftp2Files, err := sftp2.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + + osFilesByName := map[string]os.FileInfo{} + for _, f := range osfiles { + osFilesByName[f.Name()] = f + } + sftp1FilesByName := map[string]os.FileInfo{} + for _, f := range sftp1Files { + sftp1FilesByName[f.Name()] = f + } + sftp2FilesByName := map[string]os.FileInfo{} + for _, f := range sftp2Files { + sftp2FilesByName[f.Name()] = f + } + + if len(osFilesByName) != len(sftp1FilesByName) || len(sftp1FilesByName) != len(sftp2FilesByName) { + t.Fatalf("os gives %v, sftp1 gives %v, sftp2 gives %v", len(osFilesByName), len(sftp1FilesByName), len(sftp2FilesByName)) + } + + for name, osF := range osFilesByName { + sftp1F, ok := sftp1FilesByName[name] + if !ok { + t.Fatalf("%v present in os but not sftp1", name) + } + sftp2F, ok := sftp2FilesByName[name] + if !ok { + t.Fatalf("%v present in os but not sftp2", name) + } + + //t.Logf("%v: %v %v %v", name, osF, sftp1F, sftp2F) + if osF.Size() != sftp1F.Size() || sftp1F.Size() != sftp2F.Size() { + t.Fatalf("size %v %v %v", osF.Size(), sftp1F.Size(), sftp2F.Size()) + } + if osF.IsDir() != sftp1F.IsDir() || sftp1F.IsDir() != sftp2F.IsDir() { + t.Fatalf("isdir %v %v %v", osF.IsDir(), sftp1F.IsDir(), sftp2F.IsDir()) + } + if osF.ModTime().Sub(sftp1F.ModTime()) > time.Second || sftp1F.ModTime() != sftp2F.ModTime() { + t.Fatalf("modtime %v %v %v", osF.ModTime(), sftp1F.ModTime(), sftp2F.ModTime()) + } + if osF.Mode() != sftp1F.Mode() || sftp1F.Mode() != sftp2F.Mode() { + t.Fatalf("mode %x %x %x", osF.Mode(), sftp1F.Mode(), sftp2F.Mode()) + } + } +} + +var clientReadTests = []struct { + n int64 +}{ + {0}, + {1}, + {1000}, + {1024}, + {1025}, + {2048}, + {4096}, + {1 << 12}, + {1 << 13}, + {1 << 14}, + {1 << 15}, + {1 << 16}, + {1 << 17}, + {1 << 18}, + {1 << 19}, + {1 << 20}, +} + +func TestClientRead(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + d, err := ioutil.TempDir("", "sftptest") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(d) + + for _, tt := range clientReadTests { + f, err := ioutil.TempFile(d, "read-test") + if err != nil { + t.Fatal(err) + } + defer f.Close() + hash := writeN(t, f, tt.n) + f2, err := sftp.Open(f.Name()) + if err != nil { + t.Fatal(err) + } + defer f2.Close() + hash2, n := readHash(t, f2) + if hash != hash2 || tt.n != n { + t.Errorf("Read: hash: want: %q, got %q, read: want: %v, got %v", hash, hash2, tt.n, n) + } + } +} + +// readHash reads r until EOF returning the number of bytes read +// and the hash of the contents. +func readHash(t *testing.T, r io.Reader) (string, int64) { + h := sha1.New() + tr := io.TeeReader(r, h) + read, err := io.Copy(ioutil.Discard, tr) + if err != nil { + t.Fatal(err) + } + return string(h.Sum(nil)), read +} + +// writeN writes n bytes of random data to w and returns the +// hash of that data. +func writeN(t *testing.T, w io.Writer, n int64) string { + rand, err := os.Open("/dev/urandom") + if err != nil { + t.Fatal(err) + } + defer rand.Close() + + h := sha1.New() + + mw := io.MultiWriter(w, h) + + written, err := io.CopyN(mw, rand, n) + if err != nil { + t.Fatal(err) + } + if written != n { + t.Fatalf("CopyN(%v): wrote: %v", n, written) + } + return string(h.Sum(nil)) +} + +var clientWriteTests = []struct { + n int + total int64 // cumulative file size +}{ + {0, 0}, + {1, 1}, + {0, 1}, + {999, 1000}, + {24, 1024}, + {1023, 2047}, + {2048, 4095}, + {1 << 12, 8191}, + {1 << 13, 16383}, + {1 << 14, 32767}, + {1 << 15, 65535}, + {1 << 16, 131071}, + {1 << 17, 262143}, + {1 << 18, 524287}, + {1 << 19, 1048575}, + {1 << 20, 2097151}, + {1 << 21, 4194303}, +} + +func TestClientWrite(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + d, err := ioutil.TempDir("", "sftptest") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(d) + + f := path.Join(d, "writeTest") + w, err := sftp.Create(f) + if err != nil { + t.Fatal(err) + } + defer w.Close() + + for _, tt := range clientWriteTests { + got, err := w.Write(make([]byte, tt.n)) + if err != nil { + t.Fatal(err) + } + if got != tt.n { + t.Errorf("Write(%v): wrote: want: %v, got %v", tt.n, tt.n, got) + } + fi, err := os.Stat(f) + if err != nil { + t.Fatal(err) + } + if total := fi.Size(); total != tt.total { + t.Errorf("Write(%v): size: want: %v, got %v", tt.n, tt.total, total) + } + } +} + +// taken from github.com/kr/fs/walk_test.go + +type PathTest struct { + path, result string +} + +type Node struct { + name string + entries []*Node // nil if the entry is a file + mark int +} + +var tree = &Node{ + "testdata", + []*Node{ + {"a", nil, 0}, + {"b", []*Node{}, 0}, + {"c", nil, 0}, + { + "d", + []*Node{ + {"x", nil, 0}, + {"y", []*Node{}, 0}, + { + "z", + []*Node{ + {"u", nil, 0}, + {"v", nil, 0}, + }, + 0, + }, + }, + 0, + }, + }, + 0, +} + +func walkTree(n *Node, path string, f func(path string, n *Node)) { + f(path, n) + for _, e := range n.entries { + walkTree(e, filepath.Join(path, e.name), f) + } +} + +func makeTree(t *testing.T) { + walkTree(tree, tree.name, func(path string, n *Node) { + if n.entries == nil { + fd, err := os.Create(path) + if err != nil { + t.Errorf("makeTree: %v", err) + return + } + fd.Close() + } else { + os.Mkdir(path, 0770) + } + }) +} + +func markTree(n *Node) { walkTree(n, "", func(path string, n *Node) { n.mark++ }) } + +func checkMarks(t *testing.T, report bool) { + walkTree(tree, tree.name, func(path string, n *Node) { + if n.mark != 1 && report { + t.Errorf("node %s mark = %d; expected 1", path, n.mark) + } + n.mark = 0 + }) +} + +// Assumes that each node name is unique. Good enough for a test. +// If clear is true, any incoming error is cleared before return. The errors +// are always accumulated, though. +func mark(path string, info os.FileInfo, err error, errors *[]error, clear bool) error { + if err != nil { + *errors = append(*errors, err) + if clear { + return nil + } + return err + } + name := info.Name() + walkTree(tree, tree.name, func(path string, n *Node) { + if n.name == name { + n.mark++ + } + }) + return nil +} + +func TestClientWalk(t *testing.T) { + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + makeTree(t) + errors := make([]error, 0, 10) + clear := true + markFn := func(walker *fs.Walker) error { + for walker.Step() { + err := mark(walker.Path(), walker.Stat(), walker.Err(), &errors, clear) + if err != nil { + return err + } + } + return nil + } + // Expect no errors. + err := markFn(sftp.Walk(tree.name)) + if err != nil { + t.Fatalf("no error expected, found: %s", err) + } + if len(errors) != 0 { + t.Fatalf("unexpected errors: %s", errors) + } + checkMarks(t, true) + errors = errors[0:0] + + // Test permission errors. Only possible if we're not root + // and only on some file systems (AFS, FAT). To avoid errors during + // all.bash on those file systems, skip during go test -short. + if os.Getuid() > 0 && !testing.Short() { + // introduce 2 errors: chmod top-level directories to 0 + os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0) + os.Chmod(filepath.Join(tree.name, tree.entries[3].name), 0) + + // 3) capture errors, expect two. + // mark respective subtrees manually + markTree(tree.entries[1]) + markTree(tree.entries[3]) + // correct double-marking of directory itself + tree.entries[1].mark-- + tree.entries[3].mark-- + err := markFn(sftp.Walk(tree.name)) + if err != nil { + t.Fatalf("expected no error return from Walk, got %s", err) + } + if len(errors) != 2 { + t.Errorf("expected 2 errors, got %d: %s", len(errors), errors) + } + // the inaccessible subtrees were marked manually + checkMarks(t, true) + errors = errors[0:0] + + // 4) capture errors, stop after first error. + // mark respective subtrees manually + markTree(tree.entries[1]) + markTree(tree.entries[3]) + // correct double-marking of directory itself + tree.entries[1].mark-- + tree.entries[3].mark-- + clear = false // error will stop processing + err = markFn(sftp.Walk(tree.name)) + if err == nil { + t.Fatalf("expected error return from Walk") + } + if len(errors) != 1 { + t.Errorf("expected 1 error, got %d: %s", len(errors), errors) + } + // the inaccessible subtrees were marked manually + checkMarks(t, false) + errors = errors[0:0] + + // restore permissions + os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0770) + os.Chmod(filepath.Join(tree.name, tree.entries[3].name), 0770) + } + + // cleanup + if err := os.RemoveAll(tree.name); err != nil { + t.Errorf("removeTree: %v", err) + } +} + +// sftp/issue/42, abrupt server hangup would result in client hangs. +func TestServerRoughDisconnect(t *testing.T) { + if *testServerImpl { + t.Skipf("skipping with -testserver") + } + sftp, cmd := testClient(t, READONLY, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := sftp.Open("/dev/zero") + if err != nil { + t.Fatal(err) + } + defer f.Close() + go func() { + time.Sleep(100 * time.Millisecond) + cmd.Process.Kill() + }() + + io.Copy(ioutil.Discard, f) +} + +// sftp/issue/26 writing to a read only file caused client to loop. +func TestClientWriteToROFile(t *testing.T) { + sftp, cmd := testClient(t, READWRITE, NO_DELAY) + defer cmd.Wait() + defer sftp.Close() + + f, err := sftp.Open("/dev/zero") + if err != nil { + t.Fatal(err) + } + defer f.Close() + _, err = f.Write([]byte("hello")) + if err == nil { + t.Fatal("expected error, got", err) + } +} + +func benchmarkRead(b *testing.B, bufsize int, delay time.Duration) { + size := 10*1024*1024 + 123 // ~10MiB + + // open sftp client + sftp, cmd := testClient(b, READONLY, delay) + defer cmd.Wait() + defer sftp.Close() + + buf := make([]byte, bufsize) + + b.ResetTimer() + b.SetBytes(int64(size)) + + for i := 0; i < b.N; i++ { + offset := 0 + + f2, err := sftp.Open("/dev/zero") + if err != nil { + b.Fatal(err) + } + defer f2.Close() + + for offset < size { + n, err := io.ReadFull(f2, buf) + offset += n + if err == io.ErrUnexpectedEOF && offset != size { + b.Fatalf("read too few bytes! want: %d, got: %d", size, n) + } + + if err != nil { + b.Fatal(err) + } + + offset += n + } + } +} + +func BenchmarkRead1k(b *testing.B) { + benchmarkRead(b, 1*1024, NO_DELAY) +} + +func BenchmarkRead16k(b *testing.B) { + benchmarkRead(b, 16*1024, NO_DELAY) +} + +func BenchmarkRead32k(b *testing.B) { + benchmarkRead(b, 32*1024, NO_DELAY) +} + +func BenchmarkRead128k(b *testing.B) { + benchmarkRead(b, 128*1024, NO_DELAY) +} + +func BenchmarkRead512k(b *testing.B) { + benchmarkRead(b, 512*1024, NO_DELAY) +} + +func BenchmarkRead1MiB(b *testing.B) { + benchmarkRead(b, 1024*1024, NO_DELAY) +} + +func BenchmarkRead4MiB(b *testing.B) { + benchmarkRead(b, 4*1024*1024, NO_DELAY) +} + +func BenchmarkRead4MiBDelay10Msec(b *testing.B) { + benchmarkRead(b, 4*1024*1024, 10*time.Millisecond) +} + +func BenchmarkRead4MiBDelay50Msec(b *testing.B) { + benchmarkRead(b, 4*1024*1024, 50*time.Millisecond) +} + +func BenchmarkRead4MiBDelay150Msec(b *testing.B) { + benchmarkRead(b, 4*1024*1024, 150*time.Millisecond) +} + +func benchmarkWrite(b *testing.B, bufsize int, delay time.Duration) { + size := 10*1024*1024 + 123 // ~10MiB + + // open sftp client + sftp, cmd := testClient(b, false, delay) + defer cmd.Wait() + defer sftp.Close() + + data := make([]byte, size) + + b.ResetTimer() + b.SetBytes(int64(size)) + + for i := 0; i < b.N; i++ { + offset := 0 + + f, err := ioutil.TempFile("", "sftptest") + if err != nil { + b.Fatal(err) + } + defer os.Remove(f.Name()) + + f2, err := sftp.Create(f.Name()) + if err != nil { + b.Fatal(err) + } + defer f2.Close() + + for offset < size { + n, err := f2.Write(data[offset:min(len(data), offset+bufsize)]) + if err != nil { + b.Fatal(err) + } + + if offset+n < size && n != bufsize { + b.Fatalf("wrote too few bytes! want: %d, got: %d", size, n) + } + + offset += n + } + + f2.Close() + + fi, err := os.Stat(f.Name()) + if err != nil { + b.Fatal(err) + } + + if fi.Size() != int64(size) { + b.Fatalf("wrong file size: want %d, got %d", size, fi.Size()) + } + + os.Remove(f.Name()) + } +} + +func BenchmarkWrite1k(b *testing.B) { + benchmarkWrite(b, 1*1024, NO_DELAY) +} + +func BenchmarkWrite16k(b *testing.B) { + benchmarkWrite(b, 16*1024, NO_DELAY) +} + +func BenchmarkWrite32k(b *testing.B) { + benchmarkWrite(b, 32*1024, NO_DELAY) +} + +func BenchmarkWrite128k(b *testing.B) { + benchmarkWrite(b, 128*1024, NO_DELAY) +} + +func BenchmarkWrite512k(b *testing.B) { + benchmarkWrite(b, 512*1024, NO_DELAY) +} + +func BenchmarkWrite1MiB(b *testing.B) { + benchmarkWrite(b, 1024*1024, NO_DELAY) +} + +func BenchmarkWrite4MiB(b *testing.B) { + benchmarkWrite(b, 4*1024*1024, NO_DELAY) +} + +func BenchmarkWrite4MiBDelay10Msec(b *testing.B) { + benchmarkWrite(b, 4*1024*1024, 10*time.Millisecond) +} + +func BenchmarkWrite4MiBDelay50Msec(b *testing.B) { + benchmarkWrite(b, 4*1024*1024, 50*time.Millisecond) +} + +func BenchmarkWrite4MiBDelay150Msec(b *testing.B) { + benchmarkWrite(b, 4*1024*1024, 150*time.Millisecond) +} + +func benchmarkCopyDown(b *testing.B, fileSize int64, delay time.Duration) { + // Create a temp file and fill it with zero's. + src, err := ioutil.TempFile("", "sftptest") + if err != nil { + b.Fatal(err) + } + defer src.Close() + srcFilename := src.Name() + defer os.Remove(srcFilename) + zero, err := os.Open("/dev/zero") + if err != nil { + b.Fatal(err) + } + n, err := io.Copy(src, io.LimitReader(zero, fileSize)) + if err != nil { + b.Fatal(err) + } + if n < fileSize { + b.Fatal("short copy") + } + zero.Close() + src.Close() + + sftp, cmd := testClient(b, READONLY, delay) + defer cmd.Wait() + defer sftp.Close() + b.ResetTimer() + b.SetBytes(fileSize) + + for i := 0; i < b.N; i++ { + dst, err := ioutil.TempFile("", "sftptest") + if err != nil { + b.Fatal(err) + } + defer os.Remove(dst.Name()) + + src, err := sftp.Open(srcFilename) + if err != nil { + b.Fatal(err) + } + defer src.Close() + n, err := io.Copy(dst, src) + if err != nil { + b.Fatalf("copy error: %v", err) + } + if n < fileSize { + b.Fatal("unable to copy all bytes") + } + dst.Close() + fi, err := os.Stat(dst.Name()) + if err != nil { + b.Fatal(err) + } + + if fi.Size() != fileSize { + b.Fatalf("wrong file size: want %d, got %d", fileSize, fi.Size()) + } + os.Remove(dst.Name()) + } +} + +func BenchmarkCopyDown10MiBDelay10Msec(b *testing.B) { + benchmarkCopyDown(b, 10*1024*1024, 10*time.Millisecond) +} + +func BenchmarkCopyDown10MiBDelay50Msec(b *testing.B) { + benchmarkCopyDown(b, 10*1024*1024, 50*time.Millisecond) +} + +func BenchmarkCopyDown10MiBDelay150Msec(b *testing.B) { + benchmarkCopyDown(b, 10*1024*1024, 150*time.Millisecond) +} + +func benchmarkCopyUp(b *testing.B, fileSize int64, delay time.Duration) { + // Create a temp file and fill it with zero's. + src, err := ioutil.TempFile("", "sftptest") + if err != nil { + b.Fatal(err) + } + defer src.Close() + srcFilename := src.Name() + defer os.Remove(srcFilename) + zero, err := os.Open("/dev/zero") + if err != nil { + b.Fatal(err) + } + n, err := io.Copy(src, io.LimitReader(zero, fileSize)) + if err != nil { + b.Fatal(err) + } + if n < fileSize { + b.Fatal("short copy") + } + zero.Close() + src.Close() + + sftp, cmd := testClient(b, false, delay) + defer cmd.Wait() + defer sftp.Close() + + b.ResetTimer() + b.SetBytes(fileSize) + + for i := 0; i < b.N; i++ { + tmp, err := ioutil.TempFile("", "sftptest") + if err != nil { + b.Fatal(err) + } + tmp.Close() + defer os.Remove(tmp.Name()) + + dst, err := sftp.Create(tmp.Name()) + if err != nil { + b.Fatal(err) + } + defer dst.Close() + src, err := os.Open(srcFilename) + if err != nil { + b.Fatal(err) + } + defer src.Close() + n, err := io.Copy(dst, src) + if err != nil { + b.Fatalf("copy error: %v", err) + } + if n < fileSize { + b.Fatal("unable to copy all bytes") + } + + fi, err := os.Stat(tmp.Name()) + if err != nil { + b.Fatal(err) + } + + if fi.Size() != fileSize { + b.Fatalf("wrong file size: want %d, got %d", fileSize, fi.Size()) + } + os.Remove(tmp.Name()) + } +} + +func BenchmarkCopyUp10MiBDelay10Msec(b *testing.B) { + benchmarkCopyUp(b, 10*1024*1024, 10*time.Millisecond) +} + +func BenchmarkCopyUp10MiBDelay50Msec(b *testing.B) { + benchmarkCopyUp(b, 10*1024*1024, 50*time.Millisecond) +} + +func BenchmarkCopyUp10MiBDelay150Msec(b *testing.B) { + benchmarkCopyUp(b, 10*1024*1024, 150*time.Millisecond) +} diff --git a/vendor/src/github.com/pkg/sftp/client_test.go b/vendor/src/github.com/pkg/sftp/client_test.go new file mode 100644 index 000000000..8d5f0803a --- /dev/null +++ b/vendor/src/github.com/pkg/sftp/client_test.go @@ -0,0 +1,99 @@ +package sftp + +import ( + "errors" + "io" + "os" + "testing" + + "github.com/kr/fs" +) + +// assert that *Client implements fs.FileSystem +var _ fs.FileSystem = new(Client) + +// assert that *File implements io.ReadWriteCloser +var _ io.ReadWriteCloser = new(File) + +func TestNormaliseError(t *testing.T) { + var ( + ok = &StatusError{Code: ssh_FX_OK} + eof = &StatusError{Code: ssh_FX_EOF} + fail = &StatusError{Code: ssh_FX_FAILURE} + noSuchFile = &StatusError{Code: ssh_FX_NO_SUCH_FILE} + foo = errors.New("foo") + ) + + var tests = []struct { + desc string + err error + want error + }{ + { + desc: "nil error", + }, + { + desc: "not *StatusError", + err: foo, + want: foo, + }, + { + desc: "*StatusError with ssh_FX_EOF", + err: eof, + want: io.EOF, + }, + { + desc: "*StatusError with ssh_FX_NO_SUCH_FILE", + err: noSuchFile, + want: os.ErrNotExist, + }, + { + desc: "*StatusError with ssh_FX_OK", + err: ok, + }, + { + desc: "*StatusError with ssh_FX_FAILURE", + err: fail, + want: fail, + }, + } + + for _, tt := range tests { + got := normaliseError(tt.err) + if got != tt.want { + t.Errorf("normaliseError(%#v), test %q\n- want: %#v\n- got: %#v", + tt.err, tt.desc, tt.want, got) + } + } +} + +var flagsTests = []struct { + flags int + want uint32 +}{ + {os.O_RDONLY, ssh_FXF_READ}, + {os.O_WRONLY, ssh_FXF_WRITE}, + {os.O_RDWR, ssh_FXF_READ | ssh_FXF_WRITE}, + {os.O_RDWR | os.O_CREATE | os.O_TRUNC, ssh_FXF_READ | ssh_FXF_WRITE | ssh_FXF_CREAT | ssh_FXF_TRUNC}, + {os.O_WRONLY | os.O_APPEND, ssh_FXF_WRITE | ssh_FXF_APPEND}, +} + +func TestFlags(t *testing.T) { + for i, tt := range flagsTests { + got := flags(tt.flags) + if got != tt.want { + t.Errorf("test %v: flags(%x): want: %x, got: %x", i, tt.flags, tt.want, got) + } + } +} + +func TestMissingLangTag(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Fail() + } + }() + buf := marshalUint32([]byte{}, 0) + buf = marshalStatus(buf, StatusError{}) + _ = unmarshalStatus(0, buf[:len(buf)-4]) +} diff --git a/vendor/src/github.com/pkg/sftp/example_test.go b/vendor/src/github.com/pkg/sftp/example_test.go new file mode 100644 index 000000000..11dc1e12f --- /dev/null +++ b/vendor/src/github.com/pkg/sftp/example_test.go @@ -0,0 +1,90 @@ +package sftp_test + +import ( + "fmt" + "log" + "os" + "os/exec" + + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" +) + +func Example(conn *ssh.Client) { + // open an SFTP session over an existing ssh connection. + sftp, err := sftp.NewClient(conn) + if err != nil { + log.Fatal(err) + } + defer sftp.Close() + + // walk a directory + w := sftp.Walk("/home/user") + for w.Step() { + if w.Err() != nil { + continue + } + log.Println(w.Path()) + } + + // leave your mark + f, err := sftp.Create("hello.txt") + if err != nil { + log.Fatal(err) + } + if _, err := f.Write([]byte("Hello world!")); err != nil { + log.Fatal(err) + } + + // check it's there + fi, err := sftp.Lstat("hello.txt") + if err != nil { + log.Fatal(err) + } + log.Println(fi) +} + +func ExampleNewClientPipe() { + // Connect to a remote host and request the sftp subsystem via the 'ssh' + // command. This assumes that passwordless login is correctly configured. + cmd := exec.Command("ssh", "example.com", "-s", "sftp") + + // send errors from ssh to stderr + cmd.Stderr = os.Stderr + + // get stdin and stdout + wr, err := cmd.StdinPipe() + if err != nil { + log.Fatal(err) + } + rd, err := cmd.StdoutPipe() + if err != nil { + log.Fatal(err) + } + + // start the process + if err := cmd.Start(); err != nil { + log.Fatal(err) + } + defer cmd.Wait() + + // open the SFTP session + client, err := sftp.NewClientPipe(rd, wr) + if err != nil { + log.Fatal(err) + } + + // read a directory + list, err := client.ReadDir("/") + if err != nil { + log.Fatal(err) + } + + // print contents + for _, item := range list { + fmt.Println(item.Name()) + } + + // close the connection + client.Close() +} diff --git a/vendor/src/github.com/pkg/sftp/other_test.go b/vendor/src/github.com/pkg/sftp/other_test.go new file mode 100644 index 000000000..1b84ccfa8 --- /dev/null +++ b/vendor/src/github.com/pkg/sftp/other_test.go @@ -0,0 +1,5 @@ +// +build !linux,!darwin + +package sftp + +const sftpServer = "/usr/bin/false" // unsupported diff --git a/vendor/src/github.com/pkg/sftp/packet_test.go b/vendor/src/github.com/pkg/sftp/packet_test.go new file mode 100644 index 000000000..2a948dbd8 --- /dev/null +++ b/vendor/src/github.com/pkg/sftp/packet_test.go @@ -0,0 +1,345 @@ +package sftp + +import ( + "bytes" + "encoding" + "os" + "testing" +) + +var marshalUint32Tests = []struct { + v uint32 + want []byte +}{ + {1, []byte{0, 0, 0, 1}}, + {256, []byte{0, 0, 1, 0}}, + {^uint32(0), []byte{255, 255, 255, 255}}, +} + +func TestMarshalUint32(t *testing.T) { + for _, tt := range marshalUint32Tests { + got := marshalUint32(nil, tt.v) + if !bytes.Equal(tt.want, got) { + t.Errorf("marshalUint32(%d): want %v, got %v", tt.v, tt.want, got) + } + } +} + +var marshalUint64Tests = []struct { + v uint64 + want []byte +}{ + {1, []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}}, + {256, []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0}}, + {^uint64(0), []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}}, + {1 << 32, []byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0}}, +} + +func TestMarshalUint64(t *testing.T) { + for _, tt := range marshalUint64Tests { + got := marshalUint64(nil, tt.v) + if !bytes.Equal(tt.want, got) { + t.Errorf("marshalUint64(%d): want %#v, got %#v", tt.v, tt.want, got) + } + } +} + +var marshalStringTests = []struct { + v string + want []byte +}{ + {"", []byte{0, 0, 0, 0}}, + {"/foo", []byte{0x0, 0x0, 0x0, 0x4, 0x2f, 0x66, 0x6f, 0x6f}}, +} + +func TestMarshalString(t *testing.T) { + for _, tt := range marshalStringTests { + got := marshalString(nil, tt.v) + if !bytes.Equal(tt.want, got) { + t.Errorf("marshalString(%q): want %#v, got %#v", tt.v, tt.want, got) + } + } +} + +var marshalTests = []struct { + v interface{} + want []byte +}{ + {uint8(1), []byte{1}}, + {byte(1), []byte{1}}, + {uint32(1), []byte{0, 0, 0, 1}}, + {uint64(1), []byte{0, 0, 0, 0, 0, 0, 0, 1}}, + {"foo", []byte{0x0, 0x0, 0x0, 0x3, 0x66, 0x6f, 0x6f}}, + {[]uint32{1, 2, 3, 4}, []byte{0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x4}}, +} + +func TestMarshal(t *testing.T) { + for _, tt := range marshalTests { + got := marshal(nil, tt.v) + if !bytes.Equal(tt.want, got) { + t.Errorf("marshal(%v): want %#v, got %#v", tt.v, tt.want, got) + } + } +} + +var unmarshalUint32Tests = []struct { + b []byte + want uint32 + rest []byte +}{ + {[]byte{0, 0, 0, 0}, 0, nil}, + {[]byte{0, 0, 1, 0}, 256, nil}, + {[]byte{255, 0, 0, 255}, 4278190335, nil}, +} + +func TestUnmarshalUint32(t *testing.T) { + for _, tt := range unmarshalUint32Tests { + got, rest := unmarshalUint32(tt.b) + if got != tt.want || !bytes.Equal(rest, tt.rest) { + t.Errorf("unmarshalUint32(%v): want %v, %#v, got %v, %#v", tt.b, tt.want, tt.rest, got, rest) + } + } +} + +var unmarshalUint64Tests = []struct { + b []byte + want uint64 + rest []byte +}{ + {[]byte{0, 0, 0, 0, 0, 0, 0, 0}, 0, nil}, + {[]byte{0, 0, 0, 0, 0, 0, 1, 0}, 256, nil}, + {[]byte{255, 0, 0, 0, 0, 0, 0, 255}, 18374686479671623935, nil}, +} + +func TestUnmarshalUint64(t *testing.T) { + for _, tt := range unmarshalUint64Tests { + got, rest := unmarshalUint64(tt.b) + if got != tt.want || !bytes.Equal(rest, tt.rest) { + t.Errorf("unmarshalUint64(%v): want %v, %#v, got %v, %#v", tt.b, tt.want, tt.rest, got, rest) + } + } +} + +var unmarshalStringTests = []struct { + b []byte + want string + rest []byte +}{ + {marshalString(nil, ""), "", nil}, + {marshalString(nil, "blah"), "blah", nil}, +} + +func TestUnmarshalString(t *testing.T) { + for _, tt := range unmarshalStringTests { + got, rest := unmarshalString(tt.b) + if got != tt.want || !bytes.Equal(rest, tt.rest) { + t.Errorf("unmarshalUint64(%v): want %q, %#v, got %q, %#v", tt.b, tt.want, tt.rest, got, rest) + } + } +} + +var sendPacketTests = []struct { + p encoding.BinaryMarshaler + want []byte +}{ + {sshFxInitPacket{ + Version: 3, + Extensions: []extensionPair{ + {"posix-rename@openssh.com", "1"}, + }, + }, []byte{0x0, 0x0, 0x0, 0x26, 0x1, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x18, 0x70, 0x6f, 0x73, 0x69, 0x78, 0x2d, 0x72, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x40, 0x6f, 0x70, 0x65, 0x6e, 0x73, 0x73, 0x68, 0x2e, 0x63, 0x6f, 0x6d, 0x0, 0x0, 0x0, 0x1, 0x31}}, + + {sshFxpOpenPacket{ + ID: 1, + Path: "/foo", + Pflags: flags(os.O_RDONLY), + }, []byte{0x0, 0x0, 0x0, 0x15, 0x3, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x4, 0x2f, 0x66, 0x6f, 0x6f, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0}}, + + {sshFxpWritePacket{ + ID: 124, + Handle: "foo", + Offset: 13, + Length: uint32(len([]byte("bar"))), + Data: []byte("bar"), + }, []byte{0x0, 0x0, 0x0, 0x1b, 0x6, 0x0, 0x0, 0x0, 0x7c, 0x0, 0x0, 0x0, 0x3, 0x66, 0x6f, 0x6f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xd, 0x0, 0x0, 0x0, 0x3, 0x62, 0x61, 0x72}}, + + {sshFxpSetstatPacket{ + ID: 31, + Path: "/bar", + Flags: flags(os.O_WRONLY), + Attrs: struct { + UID uint32 + GID uint32 + }{1000, 100}, + }, []byte{0x0, 0x0, 0x0, 0x19, 0x9, 0x0, 0x0, 0x0, 0x1f, 0x0, 0x0, 0x0, 0x4, 0x2f, 0x62, 0x61, 0x72, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x3, 0xe8, 0x0, 0x0, 0x0, 0x64}}, +} + +func TestSendPacket(t *testing.T) { + for _, tt := range sendPacketTests { + var w bytes.Buffer + sendPacket(&w, tt.p) + if got := w.Bytes(); !bytes.Equal(tt.want, got) { + t.Errorf("sendPacket(%v): want %#v, got %#v", tt.p, tt.want, got) + } + } +} + +func sp(p encoding.BinaryMarshaler) []byte { + var w bytes.Buffer + sendPacket(&w, p) + return w.Bytes() +} + +var recvPacketTests = []struct { + b []byte + want uint8 + rest []byte +}{ + {sp(sshFxInitPacket{ + Version: 3, + Extensions: []extensionPair{ + {"posix-rename@openssh.com", "1"}, + }, + }), ssh_FXP_INIT, []byte{0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x18, 0x70, 0x6f, 0x73, 0x69, 0x78, 0x2d, 0x72, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x40, 0x6f, 0x70, 0x65, 0x6e, 0x73, 0x73, 0x68, 0x2e, 0x63, 0x6f, 0x6d, 0x0, 0x0, 0x0, 0x1, 0x31}}, +} + +func TestRecvPacket(t *testing.T) { + for _, tt := range recvPacketTests { + r := bytes.NewReader(tt.b) + got, rest, _ := recvPacket(r) + if got != tt.want || !bytes.Equal(rest, tt.rest) { + t.Errorf("recvPacket(%#v): want %v, %#v, got %v, %#v", tt.b, tt.want, tt.rest, got, rest) + } + } +} + +func TestSSHFxpOpenPacketreadonly(t *testing.T) { + var tests = []struct { + pflags uint32 + ok bool + }{ + { + pflags: ssh_FXF_READ, + ok: true, + }, + { + pflags: ssh_FXF_WRITE, + ok: false, + }, + { + pflags: ssh_FXF_READ | ssh_FXF_WRITE, + ok: false, + }, + } + + for _, tt := range tests { + p := &sshFxpOpenPacket{ + Pflags: tt.pflags, + } + + if want, got := tt.ok, p.readonly(); want != got { + t.Errorf("unexpected value for p.readonly(): want: %v, got: %v", + want, got) + } + } +} + +func TestSSHFxpOpenPackethasPflags(t *testing.T) { + var tests = []struct { + desc string + haveFlags uint32 + testFlags []uint32 + ok bool + }{ + { + desc: "have read, test against write", + haveFlags: ssh_FXF_READ, + testFlags: []uint32{ssh_FXF_WRITE}, + ok: false, + }, + { + desc: "have write, test against read", + haveFlags: ssh_FXF_WRITE, + testFlags: []uint32{ssh_FXF_READ}, + ok: false, + }, + { + desc: "have read+write, test against read", + haveFlags: ssh_FXF_READ | ssh_FXF_WRITE, + testFlags: []uint32{ssh_FXF_READ}, + ok: true, + }, + { + desc: "have read+write, test against write", + haveFlags: ssh_FXF_READ | ssh_FXF_WRITE, + testFlags: []uint32{ssh_FXF_WRITE}, + ok: true, + }, + { + desc: "have read+write, test against read+write", + haveFlags: ssh_FXF_READ | ssh_FXF_WRITE, + testFlags: []uint32{ssh_FXF_READ, ssh_FXF_WRITE}, + ok: true, + }, + } + + for _, tt := range tests { + t.Log(tt.desc) + + p := &sshFxpOpenPacket{ + Pflags: tt.haveFlags, + } + + if want, got := tt.ok, p.hasPflags(tt.testFlags...); want != got { + t.Errorf("unexpected value for p.hasPflags(%#v): want: %v, got: %v", + tt.testFlags, want, got) + } + } +} + +func BenchmarkMarshalInit(b *testing.B) { + for i := 0; i < b.N; i++ { + sp(sshFxInitPacket{ + Version: 3, + Extensions: []extensionPair{ + {"posix-rename@openssh.com", "1"}, + }, + }) + } +} + +func BenchmarkMarshalOpen(b *testing.B) { + for i := 0; i < b.N; i++ { + sp(sshFxpOpenPacket{ + ID: 1, + Path: "/home/test/some/random/path", + Pflags: flags(os.O_RDONLY), + }) + } +} + +func BenchmarkMarshalWriteWorstCase(b *testing.B) { + data := make([]byte, 32*1024) + for i := 0; i < b.N; i++ { + sp(sshFxpWritePacket{ + ID: 1, + Handle: "someopaquehandle", + Offset: 0, + Length: uint32(len(data)), + Data: data, + }) + } +} + +func BenchmarkMarshalWrite1k(b *testing.B) { + data := make([]byte, 1024) + for i := 0; i < b.N; i++ { + sp(sshFxpWritePacket{ + ID: 1, + Handle: "someopaquehandle", + Offset: 0, + Length: uint32(len(data)), + Data: data, + }) + } +} diff --git a/vendor/src/github.com/pkg/sftp/server_integration_test.go b/vendor/src/github.com/pkg/sftp/server_integration_test.go new file mode 100644 index 000000000..4d70829e5 --- /dev/null +++ b/vendor/src/github.com/pkg/sftp/server_integration_test.go @@ -0,0 +1,672 @@ +package sftp + +// sftp server integration tests +// enable with -integration +// example invokation (darwin): gofmt -w `find . -name \*.go` && (cd server_standalone/ ; go build -tags debug) && go test -tags debug github.com/pkg/sftp -integration -v -sftp /usr/libexec/sftp-server -run ServerCompareSubsystems + +import ( + "bytes" + "encoding/hex" + "flag" + "fmt" + "io/ioutil" + "math/rand" + "net" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" + "time" + + "github.com/kr/fs" + "golang.org/x/crypto/ssh" +) + +var testSftpClientBin = flag.String("sftp_client", "/usr/bin/sftp", "location of the sftp client binary") +var sshServerDebugStream = ioutil.Discard +var sftpServerDebugStream = ioutil.Discard +var sftpClientDebugStream = ioutil.Discard + +const ( + GOLANG_SFTP = true + OPENSSH_SFTP = false +) + +var ( + hostPrivateKeySigner ssh.Signer + privKey = []byte(` +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEArhp7SqFnXVZAgWREL9Ogs+miy4IU/m0vmdkoK6M97G9NX/Pj +wf8I/3/ynxmcArbt8Rc4JgkjT2uxx/NqR0yN42N1PjO5Czu0dms1PSqcKIJdeUBV +7gdrKSm9Co4d2vwfQp5mg47eG4w63pz7Drk9+VIyi9YiYH4bve7WnGDswn4ycvYZ +slV5kKnjlfCdPig+g5P7yQYud0cDWVwyA0+kxvL6H3Ip+Fu8rLDZn4/P1WlFAIuc +PAf4uEKDGGmC2URowi5eesYR7f6GN/HnBs2776laNlAVXZUmYTUfOGagwLsEkx8x +XdNqntfbs2MOOoK+myJrNtcB9pCrM0H6um19uQIDAQABAoIBABkWr9WdVKvalgkP +TdQmhu3mKRNyd1wCl+1voZ5IM9Ayac/98UAvZDiNU4Uhx52MhtVLJ0gz4Oa8+i16 +IkKMAZZW6ro/8dZwkBzQbieWUFJ2Fso2PyvB3etcnGU8/Yhk9IxBDzy+BbuqhYE2 +1ebVQtz+v1HvVZzaD11bYYm/Xd7Y28QREVfFen30Q/v3dv7dOteDE/RgDS8Czz7w +jMW32Q8JL5grz7zPkMK39BLXsTcSYcaasT2ParROhGJZDmbgd3l33zKCVc1zcj9B +SA47QljGd09Tys958WWHgtj2o7bp9v1Ufs4LnyKgzrB80WX1ovaSQKvd5THTLchO +kLIhUAECgYEA2doGXy9wMBmTn/hjiVvggR1aKiBwUpnB87Hn5xCMgoECVhFZlT6l +WmZe7R2klbtG1aYlw+y+uzHhoVDAJW9AUSV8qoDUwbRXvBVlp+In5wIqJ+VjfivK +zgIfzomL5NvDz37cvPmzqIeySTowEfbQyq7CUQSoDtE9H97E2wWZhDkCgYEAzJdJ +k+NSFoTkHhfD3L0xCDHpRV3gvaOeew8524fVtVUq53X8m91ng4AX1r74dCUYwwiF +gqTtSSJfx2iH1xKnNq28M9uKg7wOrCKrRqNPnYUO3LehZEC7rwUr26z4iJDHjjoB +uBcS7nw0LJ+0Zeg1IF+aIdZGV3MrAKnrzWPixYECgYBsffX6ZWebrMEmQ89eUtFF +u9ZxcGI/4K8ErC7vlgBD5ffB4TYZ627xzFWuBLs4jmHCeNIJ9tct5rOVYN+wRO1k +/CRPzYUnSqb+1jEgILL6istvvv+DkE+ZtNkeRMXUndWwel94BWsBnUKe0UmrSJ3G +sq23J3iCmJW2T3z+DpXbkQKBgQCK+LUVDNPE0i42NsRnm+fDfkvLP7Kafpr3Umdl +tMY474o+QYn+wg0/aPJIf9463rwMNyyhirBX/k57IIktUdFdtfPicd2MEGETElWv +nN1GzYxD50Rs2f/jKisZhEwqT9YNyV9DkgDdGGdEbJNYqbv0qpwDIg8T9foe8E1p +bdErgQKBgAt290I3L316cdxIQTkJh1DlScN/unFffITwu127WMr28Jt3mq3cZpuM +Aecey/eEKCj+Rlas5NDYKsB18QIuAw+qqWyq0LAKLiAvP1965Rkc4PLScl3MgJtO +QYa37FK0p8NcDeUuF86zXBVutwS5nJLchHhKfd590ks57OROtm29 +-----END RSA PRIVATE KEY----- +`) +) + +func init() { + var err error + hostPrivateKeySigner, err = ssh.ParsePrivateKey(privKey) + if err != nil { + panic(err) + } +} + +func keyAuth(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) { + permissions := &ssh.Permissions{ + CriticalOptions: map[string]string{}, + Extensions: map[string]string{}, + } + return permissions, nil +} + +func pwAuth(conn ssh.ConnMetadata, pw []byte) (*ssh.Permissions, error) { + permissions := &ssh.Permissions{ + CriticalOptions: map[string]string{}, + Extensions: map[string]string{}, + } + return permissions, nil +} + +func basicServerConfig() *ssh.ServerConfig { + config := ssh.ServerConfig{ + Config: ssh.Config{ + MACs: []string{"hmac-sha1"}, + }, + PasswordCallback: pwAuth, + PublicKeyCallback: keyAuth, + } + config.AddHostKey(hostPrivateKeySigner) + return &config +} + +type sshServer struct { + useSubsystem bool + conn net.Conn + config *ssh.ServerConfig + sshConn *ssh.ServerConn + newChans <-chan ssh.NewChannel + newReqs <-chan *ssh.Request +} + +func sshServerFromConn(conn net.Conn, useSubsystem bool, config *ssh.ServerConfig) (*sshServer, error) { + // From a standard TCP connection to an encrypted SSH connection + sshConn, newChans, newReqs, err := ssh.NewServerConn(conn, config) + if err != nil { + return nil, err + } + + svr := &sshServer{useSubsystem, conn, config, sshConn, newChans, newReqs} + svr.listenChannels() + return svr, nil +} + +func (svr *sshServer) Wait() error { + return svr.sshConn.Wait() +} + +func (svr *sshServer) Close() error { + return svr.sshConn.Close() +} + +func (svr *sshServer) listenChannels() { + go func() { + for chanReq := range svr.newChans { + go svr.handleChanReq(chanReq) + } + }() + go func() { + for req := range svr.newReqs { + go svr.handleReq(req) + } + }() +} + +func (svr *sshServer) handleReq(req *ssh.Request) { + switch req.Type { + default: + rejectRequest(req) + } +} + +type sshChannelServer struct { + svr *sshServer + chanReq ssh.NewChannel + ch ssh.Channel + newReqs <-chan *ssh.Request +} + +type sshSessionChannelServer struct { + *sshChannelServer + env []string +} + +func (svr *sshServer) handleChanReq(chanReq ssh.NewChannel) { + fmt.Fprintf(sshServerDebugStream, "channel request: %v, extra: '%v'\n", chanReq.ChannelType(), hex.EncodeToString(chanReq.ExtraData())) + switch chanReq.ChannelType() { + case "session": + if ch, reqs, err := chanReq.Accept(); err != nil { + fmt.Fprintf(sshServerDebugStream, "fail to accept channel request: %v\n", err) + chanReq.Reject(ssh.ResourceShortage, "channel accept failure") + } else { + chsvr := &sshSessionChannelServer{ + sshChannelServer: &sshChannelServer{svr, chanReq, ch, reqs}, + env: append([]string{}, os.Environ()...), + } + chsvr.handle() + } + default: + chanReq.Reject(ssh.UnknownChannelType, "channel type is not a session") + } +} + +func (chsvr *sshSessionChannelServer) handle() { + // should maybe do something here... + go chsvr.handleReqs() +} + +func (chsvr *sshSessionChannelServer) handleReqs() { + for req := range chsvr.newReqs { + chsvr.handleReq(req) + } + fmt.Fprintf(sshServerDebugStream, "ssh server session channel complete\n") +} + +func (chsvr *sshSessionChannelServer) handleReq(req *ssh.Request) { + switch req.Type { + case "env": + chsvr.handleEnv(req) + case "subsystem": + chsvr.handleSubsystem(req) + default: + rejectRequest(req) + } +} + +func rejectRequest(req *ssh.Request) error { + fmt.Fprintf(sshServerDebugStream, "ssh rejecting request, type: %s\n", req.Type) + err := req.Reply(false, []byte{}) + if err != nil { + fmt.Fprintf(sshServerDebugStream, "ssh request reply had error: %v\n", err) + } + return err +} + +func rejectRequestUnmarshalError(req *ssh.Request, s interface{}, err error) error { + fmt.Fprintf(sshServerDebugStream, "ssh request unmarshaling error, type '%T': %v\n", s, err) + rejectRequest(req) + return err +} + +// env request form: +type sshEnvRequest struct { + Envvar string + Value string +} + +func (chsvr *sshSessionChannelServer) handleEnv(req *ssh.Request) error { + envReq := &sshEnvRequest{} + if err := ssh.Unmarshal(req.Payload, envReq); err != nil { + return rejectRequestUnmarshalError(req, envReq, err) + } + req.Reply(true, nil) + + found := false + for i, envstr := range chsvr.env { + if strings.HasPrefix(envstr, envReq.Envvar+"=") { + found = true + chsvr.env[i] = envReq.Envvar + "=" + envReq.Value + } + } + if !found { + chsvr.env = append(chsvr.env, envReq.Envvar+"="+envReq.Value) + } + + return nil +} + +// Payload: int: command size, string: command +type sshSubsystemRequest struct { + Name string +} + +type sshSubsystemExitStatus struct { + Status uint32 +} + +func (chsvr *sshSessionChannelServer) handleSubsystem(req *ssh.Request) error { + defer func() { + err1 := chsvr.ch.CloseWrite() + err2 := chsvr.ch.Close() + fmt.Fprintf(sshServerDebugStream, "ssh server subsystem request complete, err: %v %v\n", err1, err2) + }() + + subsystemReq := &sshSubsystemRequest{} + if err := ssh.Unmarshal(req.Payload, subsystemReq); err != nil { + return rejectRequestUnmarshalError(req, subsystemReq, err) + } + + // reply to the ssh client + + // no idea if this is actually correct spec-wise. + // just enough for an sftp server to start. + if subsystemReq.Name != "sftp" { + return req.Reply(false, nil) + } + + req.Reply(true, nil) + + if !chsvr.svr.useSubsystem { + // use the openssh sftp server backend; this is to test the ssh code, not the sftp code, + // or is used for comparison between our sftp subsystem and the openssh sftp subsystem + cmd := exec.Command(*testSftp, "-e", "-l", "DEBUG") // log to stderr + cmd.Stdin = chsvr.ch + cmd.Stdout = chsvr.ch + cmd.Stderr = sftpServerDebugStream + if err := cmd.Start(); err != nil { + return err + } + return cmd.Wait() + } + + sftpServer, err := NewServer( + chsvr.ch, + chsvr.ch, + WithDebug(sftpServerDebugStream), + ) + if err != nil { + return err + } + + // wait for the session to close + runErr := sftpServer.Serve() + exitStatus := uint32(1) + if runErr == nil { + exitStatus = uint32(0) + } + + _, exitStatusErr := chsvr.ch.SendRequest("exit-status", false, ssh.Marshal(sshSubsystemExitStatus{exitStatus})) + return exitStatusErr +} + +// starts an ssh server to test. returns: host string and port +func testServer(t *testing.T, useSubsystem bool, readonly bool) (net.Listener, string, int) { + if !*testIntegration { + t.Skip("skipping intergration test") + } + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + + host, portStr, err := net.SplitHostPort(listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + port, err := strconv.Atoi(portStr) + if err != nil { + t.Fatal(err) + } + + go func() { + for { + conn, err := listener.Accept() + if err != nil { + fmt.Fprintf(sshServerDebugStream, "ssh server socket closed: %v\n", err) + break + } + + go func() { + defer conn.Close() + sshSvr, err := sshServerFromConn(conn, useSubsystem, basicServerConfig()) + if err != nil { + t.Error(err) + return + } + err = sshSvr.Wait() + fmt.Fprintf(sshServerDebugStream, "ssh server finished, err: %v\n", err) + }() + } + }() + + return listener, host, port +} + +func runSftpClient(t *testing.T, script string, path string, host string, port int) (string, error) { + // if sftp client binary is unavailable, skip test + if _, err := os.Stat(*testSftpClientBin); err != nil { + t.Skip("sftp client binary unavailable") + } + args := []string{ + // "-vvvv", + "-b", "-", + "-o", "StrictHostKeyChecking=no", + "-o", "LogLevel=ERROR", + "-o", "UserKnownHostsFile /dev/null", + "-P", fmt.Sprintf("%d", port), fmt.Sprintf("%s:%s", host, path), + } + cmd := exec.Command(*testSftpClientBin, args...) + var stdout bytes.Buffer + cmd.Stdin = bytes.NewBufferString(script) + cmd.Stdout = &stdout + cmd.Stderr = sftpClientDebugStream + if err := cmd.Start(); err != nil { + return "", err + } + err := cmd.Wait() + return string(stdout.Bytes()), err +} + +func TestServerCompareSubsystems(t *testing.T) { + listenerGo, hostGo, portGo := testServer(t, GOLANG_SFTP, READONLY) + listenerOp, hostOp, portOp := testServer(t, OPENSSH_SFTP, READONLY) + defer listenerGo.Close() + defer listenerOp.Close() + + script := ` +ls / +ls -l / +ls /dev/ +ls -l /dev/ +ls -l /etc/ +ls -l /bin/ +ls -l /usr/bin/ +` + outputGo, err := runSftpClient(t, script, "/", hostGo, portGo) + if err != nil { + t.Fatal(err) + } + + outputOp, err := runSftpClient(t, script, "/", hostOp, portOp) + if err != nil { + t.Fatal(err) + } + + newlineRegex := regexp.MustCompile(`\r*\n`) + spaceRegex := regexp.MustCompile(`\s+`) + outputGoLines := newlineRegex.Split(outputGo, -1) + outputOpLines := newlineRegex.Split(outputOp, -1) + + for i, goLine := range outputGoLines { + if i > len(outputOpLines) { + t.Fatalf("output line count differs") + } + opLine := outputOpLines[i] + bad := false + if goLine != opLine { + goWords := spaceRegex.Split(goLine, -1) + opWords := spaceRegex.Split(opLine, -1) + // allow words[2] and [3] to be different as these are users & groups + // also allow words[1] to differ as the link count for directories like + // proc is unstable during testing as processes are created/destroyed. + for j, goWord := range goWords { + if j > len(opWords) { + bad = true + } + opWord := opWords[j] + if goWord != opWord && j != 1 && j != 2 && j != 3 { + bad = true + } + } + } + + if bad { + t.Errorf("outputs differ, go:\n%v\nopenssh:\n%v\n", goLine, opLine) + } + } +} + +var rng = rand.New(rand.NewSource(time.Now().Unix())) + +func randData(length int) []byte { + data := make([]byte, length) + for i := 0; i < length; i++ { + data[i] = byte(rng.Uint32()) + } + return data +} + +func randName() string { + return "sftp." + hex.EncodeToString(randData(16)) +} + +func TestServerMkdirRmdir(t *testing.T) { + listenerGo, hostGo, portGo := testServer(t, GOLANG_SFTP, READONLY) + defer listenerGo.Close() + + tmpDir := "/tmp/" + randName() + defer os.RemoveAll(tmpDir) + + // mkdir remote + if _, err := runSftpClient(t, "mkdir "+tmpDir, "/", hostGo, portGo); err != nil { + t.Fatal(err) + } + + // directory should now exist + if _, err := os.Stat(tmpDir); err != nil { + t.Fatal(err) + } + + // now remove the directory + if _, err := runSftpClient(t, "rmdir "+tmpDir, "/", hostGo, portGo); err != nil { + t.Fatal(err) + } + + if _, err := os.Stat(tmpDir); err == nil { + t.Fatal("should have error after deleting the directory") + } +} + +func TestServerSymlink(t *testing.T) { + listenerGo, hostGo, portGo := testServer(t, GOLANG_SFTP, READONLY) + defer listenerGo.Close() + + link := "/tmp/" + randName() + defer os.RemoveAll(link) + + // now create a symbolic link within the new directory + if output, err := runSftpClient(t, "symlink /bin/sh "+link, "/", hostGo, portGo); err != nil { + t.Fatalf("failed: %v %v", err, string(output)) + } + + // symlink should now exist + if stat, err := os.Lstat(link); err != nil { + t.Fatal(err) + } else if (stat.Mode() & os.ModeSymlink) != os.ModeSymlink { + t.Fatalf("is not a symlink: %v", stat.Mode()) + } +} + +func TestServerPut(t *testing.T) { + listenerGo, hostGo, portGo := testServer(t, GOLANG_SFTP, READONLY) + defer listenerGo.Close() + + tmpFileLocal := "/tmp/" + randName() + tmpFileRemote := "/tmp/" + randName() + defer os.RemoveAll(tmpFileLocal) + defer os.RemoveAll(tmpFileRemote) + + t.Logf("put: local %v remote %v", tmpFileLocal, tmpFileRemote) + + // create a file with random contents. This will be the local file pushed to the server + tmpFileLocalData := randData(10 * 1024 * 1024) + if err := ioutil.WriteFile(tmpFileLocal, tmpFileLocalData, 0644); err != nil { + t.Fatal(err) + } + + // sftp the file to the server + if output, err := runSftpClient(t, "put "+tmpFileLocal+" "+tmpFileRemote, "/", hostGo, portGo); err != nil { + t.Fatalf("runSftpClient failed: %v, output\n%v\n", err, output) + } + + // tmpFile2 should now exist, with the same contents + if tmpFileRemoteData, err := ioutil.ReadFile(tmpFileRemote); err != nil { + t.Fatal(err) + } else if string(tmpFileLocalData) != string(tmpFileRemoteData) { + t.Fatal("contents of file incorrect after put") + } +} + +func TestServerGet(t *testing.T) { + listenerGo, hostGo, portGo := testServer(t, GOLANG_SFTP, READONLY) + defer listenerGo.Close() + + tmpFileLocal := "/tmp/" + randName() + tmpFileRemote := "/tmp/" + randName() + defer os.RemoveAll(tmpFileLocal) + defer os.RemoveAll(tmpFileRemote) + + t.Logf("get: local %v remote %v", tmpFileLocal, tmpFileRemote) + + // create a file with random contents. This will be the remote file pulled from the server + tmpFileRemoteData := randData(10 * 1024 * 1024) + if err := ioutil.WriteFile(tmpFileRemote, tmpFileRemoteData, 0644); err != nil { + t.Fatal(err) + } + + // sftp the file to the server + if output, err := runSftpClient(t, "get "+tmpFileRemote+" "+tmpFileLocal, "/", hostGo, portGo); err != nil { + t.Fatalf("runSftpClient failed: %v, output\n%v\n", err, output) + } + + // tmpFile2 should now exist, with the same contents + if tmpFileLocalData, err := ioutil.ReadFile(tmpFileLocal); err != nil { + t.Fatal(err) + } else if string(tmpFileLocalData) != string(tmpFileRemoteData) { + t.Fatal("contents of file incorrect after put") + } +} + +func compareDirectoriesRecursive(t *testing.T, aroot, broot string) { + walker := fs.Walk(aroot) + for walker.Step() { + if err := walker.Err(); err != nil { + t.Fatal(err) + } + // find paths + aPath := walker.Path() + aRel, err := filepath.Rel(aroot, aPath) + if err != nil { + t.Fatalf("could not find relative path for %v: %v", aPath, err) + } + bPath := path.Join(broot, aRel) + + if aRel == "." { + continue + } + + //t.Logf("comparing: %v a: %v b %v", aRel, aPath, bPath) + + // if a is a link, the sftp recursive copy won't have copied it. ignore + aLink, err := os.Lstat(aPath) + if err != nil { + t.Fatalf("could not lstat %v: %v", aPath, err) + } + if aLink.Mode()&os.ModeSymlink != 0 { + continue + } + + // stat the files + aFile, err := os.Stat(aPath) + if err != nil { + t.Fatalf("could not stat %v: %v", aPath, err) + } + bFile, err := os.Stat(bPath) + if err != nil { + t.Fatalf("could not stat %v: %v", bPath, err) + } + + // compare stats, with some leniency for the timestamp + if aFile.Mode() != bFile.Mode() { + t.Fatalf("modes different for %v: %v vs %v", aRel, aFile.Mode(), bFile.Mode()) + } + if !aFile.IsDir() { + if aFile.Size() != bFile.Size() { + t.Fatalf("sizes different for %v: %v vs %v", aRel, aFile.Size(), bFile.Size()) + } + } + timeDiff := aFile.ModTime().Sub(bFile.ModTime()) + if timeDiff > time.Second || timeDiff < -time.Second { + t.Fatalf("mtimes different for %v: %v vs %v", aRel, aFile.ModTime(), bFile.ModTime()) + } + + // compare contents + if !aFile.IsDir() { + if aContents, err := ioutil.ReadFile(aPath); err != nil { + t.Fatal(err) + } else if bContents, err := ioutil.ReadFile(bPath); err != nil { + t.Fatal(err) + } else if string(aContents) != string(bContents) { + t.Fatalf("contents different for %v", aRel) + } + } + } +} + +func TestServerPutRecursive(t *testing.T) { + listenerGo, hostGo, portGo := testServer(t, GOLANG_SFTP, READONLY) + defer listenerGo.Close() + + dirLocal, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + tmpDirRemote := "/tmp/" + randName() + defer os.RemoveAll(tmpDirRemote) + + t.Logf("put recursive: local %v remote %v", dirLocal, tmpDirRemote) + + // push this directory (source code etc) recursively to the server + if output, err := runSftpClient(t, "mkdir "+tmpDirRemote+"\r\nput -r -P "+dirLocal+"/ "+tmpDirRemote+"/", "/", hostGo, portGo); err != nil { + t.Fatalf("runSftpClient failed: %v, output\n%v\n", err, output) + } + + compareDirectoriesRecursive(t, dirLocal, path.Join(tmpDirRemote, path.Base(dirLocal))) +} + +func TestServerGetRecursive(t *testing.T) { + listenerGo, hostGo, portGo := testServer(t, GOLANG_SFTP, READONLY) + defer listenerGo.Close() + + dirRemote, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + tmpDirLocal := "/tmp/" + randName() + defer os.RemoveAll(tmpDirLocal) + + t.Logf("get recursive: local %v remote %v", tmpDirLocal, dirRemote) + + // pull this directory (source code etc) recursively from the server + if output, err := runSftpClient(t, "lmkdir "+tmpDirLocal+"\r\nget -r -P "+dirRemote+"/ "+tmpDirLocal+"/", "/", hostGo, portGo); err != nil { + t.Fatalf("runSftpClient failed: %v, output\n%v\n", err, output) + } + + compareDirectoriesRecursive(t, dirRemote, path.Join(tmpDirLocal, path.Base(dirRemote))) +} diff --git a/vendor/src/github.com/restic/chunker/.travis.yml b/vendor/src/github.com/restic/chunker/.travis.yml deleted file mode 100644 index 5a6bbcbeb..000000000 --- a/vendor/src/github.com/restic/chunker/.travis.yml +++ /dev/null @@ -1,11 +0,0 @@ -language: go -sudo: false - -go: - - 1.3.3 - - 1.4.2 - - 1.5.1 - -os: - - linux - - osx diff --git a/vendor/src/golang.org/x/crypto/pbkdf2/pbkdf2.go b/vendor/src/golang.org/x/crypto/pbkdf2/pbkdf2.go index c02b4d5a7..593f65300 100644 --- a/vendor/src/golang.org/x/crypto/pbkdf2/pbkdf2.go +++ b/vendor/src/golang.org/x/crypto/pbkdf2/pbkdf2.go @@ -16,7 +16,7 @@ Hash Functions SHA-1, SHA-224, SHA-256, SHA-384 and SHA-512 for HMAC. To choose, you can pass the `New` functions from the different SHA packages to pbkdf2.Key. */ -package pbkdf2 +package pbkdf2 // import "golang.org/x/crypto/pbkdf2" import ( "crypto/hmac" diff --git a/vendor/src/golang.org/x/crypto/poly1305/poly1305.go b/vendor/src/golang.org/x/crypto/poly1305/poly1305.go index 2270d2b38..4a5f826f7 100644 --- a/vendor/src/golang.org/x/crypto/poly1305/poly1305.go +++ b/vendor/src/golang.org/x/crypto/poly1305/poly1305.go @@ -16,7 +16,7 @@ used with a fixed key in order to generate one-time keys from an nonce. However, in this package AES isn't used and the one-time key is specified directly. */ -package poly1305 +package poly1305 // import "golang.org/x/crypto/poly1305" import "crypto/subtle" diff --git a/vendor/src/golang.org/x/crypto/scrypt/scrypt.go b/vendor/src/golang.org/x/crypto/scrypt/scrypt.go index 30737b0a6..dc0124b1f 100644 --- a/vendor/src/golang.org/x/crypto/scrypt/scrypt.go +++ b/vendor/src/golang.org/x/crypto/scrypt/scrypt.go @@ -5,7 +5,7 @@ // Package scrypt implements the scrypt key derivation function as defined in // Colin Percival's paper "Stronger Key Derivation via Sequential Memory-Hard // Functions" (http://www.tarsnap.com/scrypt/scrypt.pdf). -package scrypt +package scrypt // import "golang.org/x/crypto/scrypt" import ( "crypto/sha256" diff --git a/vendor/src/golang.org/x/crypto/ssh/agent/client.go b/vendor/src/golang.org/x/crypto/ssh/agent/client.go index c98cb16b2..31a0120c9 100644 --- a/vendor/src/golang.org/x/crypto/ssh/agent/client.go +++ b/vendor/src/golang.org/x/crypto/ssh/agent/client.go @@ -8,7 +8,7 @@ References: [PROTOCOL.agent]: http://cvsweb.openbsd.org/cgi-bin/cvsweb/src/usr.bin/ssh/PROTOCOL.agent?rev=HEAD */ -package agent +package agent // import "golang.org/x/crypto/ssh/agent" import ( "bytes" diff --git a/vendor/src/golang.org/x/crypto/ssh/doc.go b/vendor/src/golang.org/x/crypto/ssh/doc.go index a5ff8af6b..d6be89466 100644 --- a/vendor/src/golang.org/x/crypto/ssh/doc.go +++ b/vendor/src/golang.org/x/crypto/ssh/doc.go @@ -15,4 +15,4 @@ References: [PROTOCOL.certkeys]: http://cvsweb.openbsd.org/cgi-bin/cvsweb/src/usr.bin/ssh/PROTOCOL.certkeys?rev=HEAD [SSH-PARAMETERS]: http://www.iana.org/assignments/ssh-parameters/ssh-parameters.xml#ssh-parameters-1 */ -package ssh +package ssh // import "golang.org/x/crypto/ssh" diff --git a/vendor/src/golang.org/x/crypto/ssh/terminal/util.go b/vendor/src/golang.org/x/crypto/ssh/terminal/util.go index 0763c9a97..598e3df77 100644 --- a/vendor/src/golang.org/x/crypto/ssh/terminal/util.go +++ b/vendor/src/golang.org/x/crypto/ssh/terminal/util.go @@ -14,7 +14,7 @@ // panic(err) // } // defer terminal.Restore(0, oldState) -package terminal +package terminal // import "golang.org/x/crypto/ssh/terminal" import ( "io" diff --git a/vendor/src/golang.org/x/crypto/ssh/test/doc.go b/vendor/src/golang.org/x/crypto/ssh/test/doc.go index 29ad65e96..3f9b3346d 100644 --- a/vendor/src/golang.org/x/crypto/ssh/test/doc.go +++ b/vendor/src/golang.org/x/crypto/ssh/test/doc.go @@ -4,4 +4,4 @@ // This package contains integration tests for the // golang.org/x/crypto/ssh package. -package test +package test // import "golang.org/x/crypto/ssh/test" diff --git a/vendor/src/golang.org/x/crypto/ssh/testdata/doc.go b/vendor/src/golang.org/x/crypto/ssh/testdata/doc.go index ae7bd8b89..fcae47ca6 100644 --- a/vendor/src/golang.org/x/crypto/ssh/testdata/doc.go +++ b/vendor/src/golang.org/x/crypto/ssh/testdata/doc.go @@ -5,4 +5,4 @@ // This package contains test data shared between the various subpackages of // the golang.org/x/crypto/ssh package. Under no circumstance should // this data be used for production code. -package testdata +package testdata // import "golang.org/x/crypto/ssh/testdata" diff --git a/vendor/src/golang.org/x/net/context/context.go b/vendor/src/golang.org/x/net/context/context.go index ef2f3e86f..e7ee376c4 100644 --- a/vendor/src/golang.org/x/net/context/context.go +++ b/vendor/src/golang.org/x/net/context/context.go @@ -34,7 +34,7 @@ // // See http://blog.golang.org/context for example code for a server that uses // Contexts. -package context +package context // import "golang.org/x/net/context" import ( "errors" diff --git a/vendor/src/golang.org/x/net/context/ctxhttp/ctxhttp.go b/vendor/src/golang.org/x/net/context/ctxhttp/ctxhttp.go index 9f3488813..504dd63ed 100644 --- a/vendor/src/golang.org/x/net/context/ctxhttp/ctxhttp.go +++ b/vendor/src/golang.org/x/net/context/ctxhttp/ctxhttp.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. // Package ctxhttp provides helper functions for performing context-aware HTTP requests. -package ctxhttp +package ctxhttp // import "golang.org/x/net/context/ctxhttp" import ( "io"