From 171cd0dfe1a3279d88cddd2b4a6d15a2fe816811 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 12:46:20 +0100 Subject: [PATCH 01/56] Add backend.Handle, add comments --- backend/interface.go | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/backend/interface.go b/backend/interface.go index 63a3a95f8..0872188d1 100644 --- a/backend/interface.go +++ b/backend/interface.go @@ -1,10 +1,14 @@ package backend -import "io" +import ( + "fmt" + "io" +) // Type is the type of a Blob. type Type string +// These are the different data types a backend can store. const ( Data Type = "data" Key = "key" @@ -14,10 +18,24 @@ const ( Config = "config" ) -// A Backend manages data stored somewhere. +// Handle is used to store and access data in a backend. +type Handle struct { + Type Type + Name string +} + +func (h Handle) String() string { + name := h.Name + if len(name) > 10 { + name = name[:10] + } + return fmt.Sprintf("<%s/%s>", h.Type, name) +} + +// Backend is used to store and access data. type Backend interface { - // Location returns a string that specifies the location of the repository, - // like a URL. + // Location returns a string that describes the type and location of the + // repository. Location() string // Create creates a new Blob. The data is available only after Finalize() @@ -43,6 +61,7 @@ type Backend interface { Lister } +// Lister implements listing data items stored in a backend. type Lister interface { // List returns a channel that yields all names of blobs of type t in // lexicographic order. A goroutine is started for this. If the channel @@ -50,11 +69,13 @@ type Lister interface { List(t Type, done <-chan struct{}) <-chan string } +// Deleter are backends that allow to self-delete all content stored in them. type Deleter interface { // Delete the complete repository. Delete() error } +// Blob is old. type Blob interface { io.Writer From d3a6e2a9915f48aa80208ed205b77a8097fa9253 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 12:47:16 +0100 Subject: [PATCH 02/56] Drop requirement from List() Closes #305 --- backend/interface.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/interface.go b/backend/interface.go index 0872188d1..5739aabe9 100644 --- a/backend/interface.go +++ b/backend/interface.go @@ -63,9 +63,9 @@ type Backend interface { // Lister implements listing data items stored in a backend. type Lister interface { - // List returns a channel that yields all names of blobs of type t in - // lexicographic order. A goroutine is started for this. If the channel - // done is closed, sending stops. + // List returns a channel that yields all names of blobs of type t in an + // arbitrary order. A goroutine is started for this. If the channel done is + // closed, sending stops. List(t Type, done <-chan struct{}) <-chan string } From 8b7bf8691d609e5d08bb6fdf8f1f32977d784ca4 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 13:13:05 +0100 Subject: [PATCH 03/56] backend: Remove Get() This is the first commit that removes the (redundant) Get() method of the backend interface. Get(x, y) is equivalent to GetReader(x, y, 0, 0). --- backend/backend_test.go | 18 +++++------------- backend/interface.go | 3 --- backend/local/local.go | 14 -------------- backend/mem_backend.go | 21 --------------------- backend/mock_backend.go | 9 --------- backend/s3/s3.go | 13 ------------- backend/sftp/sftp.go | 12 ------------ checker/checker.go | 2 +- checker/checker_test.go | 13 ------------- cmd/restic/cmd_cat.go | 4 ++-- repository/key.go | 2 +- repository/repository.go | 6 +++--- 12 files changed, 12 insertions(+), 105 deletions(-) diff --git a/backend/backend_test.go b/backend/backend_test.go index 66b7b3ea3..297170f56 100644 --- a/backend/backend_test.go +++ b/backend/backend_test.go @@ -17,7 +17,7 @@ import ( func testBackendConfig(b backend.Backend, t *testing.T) { // create config and read it back - _, err := b.Get(backend.Config, "") + _, err := b.GetReader(backend.Config, "", 0, 0) Assert(t, err != nil, "did not get expected error for non-existing config") blob, err := b.Create() @@ -30,7 +30,7 @@ func testBackendConfig(b backend.Backend, t *testing.T) { // try accessing the config with different names, should all return the // same config for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} { - rd, err := b.Get(backend.Config, name) + rd, err := b.GetReader(backend.Config, name, 0, 0) Assert(t, err == nil, "unable to read config") buf, err := ioutil.ReadAll(rd) @@ -116,7 +116,7 @@ func testWrite(b backend.Backend, t testing.TB) { name := fmt.Sprintf("%s-%d", id, i) OK(t, blob.Finalize(backend.Data, name)) - rd, err := b.Get(backend.Data, name) + rd, err := b.GetReader(backend.Data, name, 0, 0) OK(t, err) buf, err := ioutil.ReadAll(rd) @@ -169,7 +169,7 @@ func testBackend(b backend.Backend, t *testing.T) { Assert(t, !ret, "blob was found to exist before creating") // try to open not existing blob - _, err = b.Get(tpe, id.String()) + _, err = b.GetReader(tpe, id.String(), 0, 0) Assert(t, err != nil, "blob data could be extracted before creation") // try to read not existing blob @@ -186,16 +186,8 @@ func testBackend(b backend.Backend, t *testing.T) { for _, test := range TestStrings { store(t, b, tpe, []byte(test.data)) - // test Get() - rd, err := b.Get(tpe, test.id) - OK(t, err) - Assert(t, rd != nil, "Get() returned nil") - - read(t, rd, []byte(test.data)) - OK(t, rd.Close()) - // test GetReader() - rd, err = b.GetReader(tpe, test.id, 0, uint(len(test.data))) + rd, err := b.GetReader(tpe, test.id, 0, uint(len(test.data))) OK(t, err) Assert(t, rd != nil, "GetReader() returned nil") diff --git a/backend/interface.go b/backend/interface.go index 5739aabe9..eccd58fde 100644 --- a/backend/interface.go +++ b/backend/interface.go @@ -42,9 +42,6 @@ type Backend interface { // has been called on the returned Blob. Create() (Blob, error) - // Get returns an io.ReadCloser for the Blob with the given name of type t. - Get(t Type, name string) (io.ReadCloser, error) - // GetReader returns an io.ReadCloser for the Blob with the given name of // type t at offset and length. GetReader(t Type, name string, offset, length uint) (io.ReadCloser, error) diff --git a/backend/local/local.go b/backend/local/local.go index d8291a8bd..80c4eefbc 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -196,20 +196,6 @@ func dirname(base string, t backend.Type, name string) string { return filepath.Join(base, n) } -// Get returns a reader that yields the content stored under the given -// name. The reader should be closed after draining it. -func (b *Local) Get(t backend.Type, name string) (io.ReadCloser, error) { - file, err := os.Open(filename(b.p, t, name)) - if err != nil { - return nil, err - } - b.mu.Lock() - open, _ := b.open[filename(b.p, t, name)] - b.open[filename(b.p, t, name)] = append(open, file) - b.mu.Unlock() - return file, nil -} - // GetReader returns an io.ReadCloser for the Blob with the given name of // type t at offset and length. If length is 0, the reader reads until EOF. func (b *Local) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { diff --git a/backend/mem_backend.go b/backend/mem_backend.go index e757566e6..e689584c2 100644 --- a/backend/mem_backend.go +++ b/backend/mem_backend.go @@ -41,10 +41,6 @@ func NewMemoryBackend() *MemoryBackend { return memCreate(be) } - be.MockBackend.GetFn = func(t Type, name string) (io.ReadCloser, error) { - return memGet(be, t, name) - } - be.MockBackend.GetReaderFn = func(t Type, name string, offset, length uint) (io.ReadCloser, error) { return memGetReader(be, t, name, offset, length) } @@ -143,23 +139,6 @@ func (rd readCloser) Close() error { return nil } -func memGet(be *MemoryBackend, t Type, name string) (io.ReadCloser, error) { - be.m.Lock() - defer be.m.Unlock() - - if t == Config { - name = "" - } - - debug.Log("MemoryBackend.Get", "get %v %v", t, name) - - if _, ok := be.data[entry{t, name}]; !ok { - return nil, errors.New("no such data") - } - - return readCloser{bytes.NewReader(be.data[entry{t, name}])}, nil -} - func memGetReader(be *MemoryBackend, t Type, name string, offset, length uint) (io.ReadCloser, error) { be.m.Lock() defer be.m.Unlock() diff --git a/backend/mock_backend.go b/backend/mock_backend.go index 92d5521c7..0d24a3bbf 100644 --- a/backend/mock_backend.go +++ b/backend/mock_backend.go @@ -10,7 +10,6 @@ import ( type MockBackend struct { CloseFn func() error CreateFn func() (Blob, error) - GetFn func(Type, string) (io.ReadCloser, error) GetReaderFn func(Type, string, uint, uint) (io.ReadCloser, error) ListFn func(Type, <-chan struct{}) <-chan string RemoveFn func(Type, string) error @@ -43,14 +42,6 @@ func (m *MockBackend) Create() (Blob, error) { return m.CreateFn() } -func (m *MockBackend) Get(t Type, name string) (io.ReadCloser, error) { - if m.GetFn == nil { - return nil, errors.New("not implemented") - } - - return m.GetFn(t, name) -} - func (m *MockBackend) GetReader(t Type, name string, offset, len uint) (io.ReadCloser, error) { if m.GetReaderFn == nil { return nil, errors.New("not implemented") diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 87b41e72e..619d41546 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -145,19 +145,6 @@ func (be *S3Backend) Create() (backend.Blob, error) { return &blob, nil } -// Get returns a reader that yields the content stored under the given -// name. The reader should be closed after draining it. -func (be *S3Backend) Get(t backend.Type, name string) (io.ReadCloser, error) { - path := s3path(t, name) - rc, err := be.client.GetObject(be.bucketname, path) - debug.Log("s3.Get", "%v %v -> err %v", t, name, err) - if err != nil { - return nil, err - } - - return rc, nil -} - // GetReader returns an io.ReadCloser for the Blob with the given name of // type t at offset and length. If length is 0, the reader reads until EOF. func (be *S3Backend) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index f52c85c53..2fd57b83c 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -344,18 +344,6 @@ func (r *SFTP) dirname(t backend.Type, name string) string { return Join(r.p, n) } -// Get returns a reader that yields the content stored under the given -// name. The reader should be closed after draining it. -func (r *SFTP) Get(t backend.Type, name string) (io.ReadCloser, error) { - // try to open file - file, err := r.c.Open(r.filename(t, name)) - if err != nil { - return nil, err - } - - return file, nil -} - // GetReader returns an io.ReadCloser for the Blob with the given name of // type t at offset and length. If length is 0, the reader reads until EOF. func (r *SFTP) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { diff --git a/checker/checker.go b/checker/checker.go index ecd2ed626..97577285d 100644 --- a/checker/checker.go +++ b/checker/checker.go @@ -647,7 +647,7 @@ func (c *Checker) CountPacks() uint64 { // checkPack reads a pack and checks the integrity of all blobs. func checkPack(r *repository.Repository, id backend.ID) error { debug.Log("Checker.checkPack", "checking pack %v", id.Str()) - rd, err := r.Backend().Get(backend.Data, id.String()) + rd, err := r.Backend().GetReader(backend.Data, id.String(), 0, 0) if err != nil { return err } diff --git a/checker/checker_test.go b/checker/checker_test.go index 99e89a22a..9ce2db5ca 100644 --- a/checker/checker_test.go +++ b/checker/checker_test.go @@ -214,19 +214,6 @@ type errorBackend struct { backend.Backend } -func (b errorBackend) Get(t backend.Type, name string) (io.ReadCloser, error) { - rd, err := b.Backend.Get(t, name) - if err != nil { - return rd, err - } - - if t != backend.Data { - return rd, err - } - - return backend.ReadCloser(faultReader{rd}), nil -} - func (b errorBackend) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { rd, err := b.Backend.GetReader(t, name, offset, length) if err != nil { diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index c8d7bffd8..2106fa306 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -101,7 +101,7 @@ func (cmd CmdCat) Execute(args []string) error { return nil case "key": - rd, err := repo.Backend().Get(backend.Key, id.String()) + rd, err := repo.Backend().GetReader(backend.Key, id.String(), 0, 0) if err != nil { return err } @@ -153,7 +153,7 @@ func (cmd CmdCat) Execute(args []string) error { switch tpe { case "pack": - rd, err := repo.Backend().Get(backend.Data, id.String()) + rd, err := repo.Backend().GetReader(backend.Data, id.String(), 0, 0) if err != nil { return err } diff --git a/repository/key.go b/repository/key.go index 22ed2ca2e..5759849fa 100644 --- a/repository/key.go +++ b/repository/key.go @@ -120,7 +120,7 @@ func SearchKey(s *Repository, password string) (*Key, error) { // LoadKey loads a key from the backend. func LoadKey(s *Repository, name string) (k *Key, err error) { // extract data from repo - rd, err := s.be.Get(backend.Key, name) + rd, err := s.be.GetReader(backend.Key, name, 0, 0) if err != nil { return nil, err } diff --git a/repository/repository.go b/repository/repository.go index bc5e380ac..a0d7f71ef 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -56,7 +56,7 @@ func (r *Repository) PrefixLength(t backend.Type) (int, error) { func (r *Repository) LoadAndDecrypt(t backend.Type, id backend.ID) ([]byte, error) { debug.Log("Repo.Load", "load %v with id %v", t, id.Str()) - rd, err := r.be.Get(t, id.String()) + rd, err := r.be.GetReader(t, id.String(), 0, 0) if err != nil { debug.Log("Repo.Load", "error loading %v: %v", id.Str(), err) return nil, err @@ -157,7 +157,7 @@ func closeOrErr(cl io.Closer, err *error) { // the item. func (r *Repository) LoadJSONUnpacked(t backend.Type, id backend.ID, item interface{}) (err error) { // load blob from backend - rd, err := r.be.Get(t, id.String()) + rd, err := r.be.GetReader(t, id.String(), 0, 0) if err != nil { return err } @@ -548,7 +548,7 @@ func LoadIndex(repo *Repository, id string) (*Index, error) { // GetDecryptReader opens the file id stored in the backend and returns a // reader that yields the decrypted content. The reader must be closed. func (r *Repository) GetDecryptReader(t backend.Type, id string) (io.ReadCloser, error) { - rd, err := r.be.Get(t, id) + rd, err := r.be.GetReader(t, id, 0, 0) if err != nil { return nil, err } From 0a24261afb5fb3fec8ba3384866197eea94b1dde Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 14:12:12 +0100 Subject: [PATCH 04/56] Add Load() for all existing backends --- backend/interface.go | 4 ++++ backend/local/local.go | 25 +++++++++++++++++++++++++ backend/mock_backend.go | 9 +++++++++ backend/s3/s3.go | 21 +++++++++++++++++++++ backend/sftp/sftp.go | 25 +++++++++++++++++++++++++ 5 files changed, 84 insertions(+) diff --git a/backend/interface.go b/backend/interface.go index eccd58fde..73c3e82b6 100644 --- a/backend/interface.go +++ b/backend/interface.go @@ -56,6 +56,10 @@ type Backend interface { Close() error Lister + + // Load returns the data stored in the backend for h at the given offset + // and saves it in p. Load has the same semantics as io.ReaderAt. + Load(h Handle, p []byte, off int64) (int, error) } // Lister implements listing data items stored in a backend. diff --git a/backend/local/local.go b/backend/local/local.go index 80c4eefbc..9946ddc4a 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -223,6 +223,31 @@ func (b *Local) GetReader(t backend.Type, name string, offset, length uint) (io. return backend.LimitReadCloser(f, int64(length)), nil } +// Load returns the data stored in the backend for h at the given offset +// and saves it in p. Load has the same semantics as io.ReaderAt. +func (b *Local) Load(h backend.Handle, p []byte, off int64) (n int, err error) { + f, err := os.Open(filename(b.p, h.Type, h.Name)) + if err != nil { + return 0, err + } + + defer func() { + e := f.Close() + if err == nil && e != nil { + err = e + } + }() + + if off > 0 { + _, err = f.Seek(off, 0) + if err != nil { + return 0, err + } + } + + return io.ReadFull(f, p) +} + // Test returns true if a blob of the given type and name exists in the backend. func (b *Local) Test(t backend.Type, name string) (bool, error) { _, err := os.Stat(filename(b.p, t, name)) diff --git a/backend/mock_backend.go b/backend/mock_backend.go index 0d24a3bbf..a5e21b8a4 100644 --- a/backend/mock_backend.go +++ b/backend/mock_backend.go @@ -10,6 +10,7 @@ import ( type MockBackend struct { CloseFn func() error CreateFn func() (Blob, error) + LoadFn func(h Handle, p []byte, off int64) (int, error) GetReaderFn func(Type, string, uint, uint) (io.ReadCloser, error) ListFn func(Type, <-chan struct{}) <-chan string RemoveFn func(Type, string) error @@ -42,6 +43,14 @@ func (m *MockBackend) Create() (Blob, error) { return m.CreateFn() } +func (m *MockBackend) Load(h Handle, p []byte, off int64) (int, error) { + if m.LoadFn == nil { + return 0, errors.New("not implemented") + } + + return m.Load(h, p, off) +} + func (m *MockBackend) GetReader(t Type, name string, offset, len uint) (io.ReadCloser, error) { if m.GetReaderFn == nil { return nil, errors.New("not implemented") diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 619d41546..2d3f89606 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -170,6 +170,27 @@ func (be *S3Backend) GetReader(t backend.Type, name string, offset, length uint) return backend.LimitReadCloser(obj, int64(length)), nil } +// Load returns the data stored in the backend for h at the given offset +// and saves it in p. Load has the same semantics as io.ReaderAt. +func (be S3Backend) Load(h backend.Handle, p []byte, off int64) (int, error) { + debug.Log("s3.Load", "%v, offset %v, len %v", h, off, len(p)) + path := s3path(h.Type, h.Name) + obj, err := be.client.GetObject(be.bucketname, path) + if err != nil { + debug.Log("s3.GetReader", " err %v", err) + return 0, err + } + + if off > 0 { + _, err = obj.Seek(off, 0) + if err != nil { + return 0, err + } + } + + return io.ReadFull(obj, p) +} + // Test returns true if a blob of the given type and name exists in the backend. func (be *S3Backend) Test(t backend.Type, name string) (bool, error) { found := false diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 2fd57b83c..007e2088a 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -366,6 +366,31 @@ func (r *SFTP) GetReader(t backend.Type, name string, offset, length uint) (io.R return backend.LimitReadCloser(f, int64(length)), nil } +// Load returns the data stored in the backend for h at the given offset +// and saves it in p. Load has the same semantics as io.ReaderAt. +func (r *SFTP) Load(h backend.Handle, p []byte, off int64) (n int, err error) { + f, err := r.c.Open(r.filename(h.Type, h.Name)) + if err != nil { + return 0, err + } + + defer func() { + e := f.Close() + if err == nil && e != nil { + err = e + } + }() + + if off > 0 { + _, err = f.Seek(off, 0) + if err != nil { + return 0, err + } + } + + return io.ReadFull(f, p) +} + // Test returns true if a blob of the given type and name exists in the backend. func (r *SFTP) Test(t backend.Type, name string) (bool, error) { _, err := r.c.Lstat(r.filename(t, name)) From 9a490f9e01523ad1344303ab9dbf41bb7a45f77f Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 17:08:03 +0100 Subject: [PATCH 05/56] Implement package-local tests --- backend/handle.go | 48 +++ backend/handle_test.go | 28 ++ backend/interface.go | 19 +- backend/local/backend_test.go | 20 + backend/local/local.go | 4 + backend/local/local_test.go | 53 +++ backend/test/generate_backend_tests.go | 111 +++++ backend/test/tests.go | 550 +++++++++++++++++++++++++ backend/test/tests_test.go | 3 + 9 files changed, 818 insertions(+), 18 deletions(-) create mode 100644 backend/handle.go create mode 100644 backend/handle_test.go create mode 100644 backend/local/backend_test.go create mode 100644 backend/local/local_test.go create mode 100644 backend/test/generate_backend_tests.go create mode 100644 backend/test/tests.go create mode 100644 backend/test/tests_test.go diff --git a/backend/handle.go b/backend/handle.go new file mode 100644 index 000000000..9c7a443ff --- /dev/null +++ b/backend/handle.go @@ -0,0 +1,48 @@ +package backend + +import ( + "errors" + "fmt" +) + +// Handle is used to store and access data in a backend. +type Handle struct { + Type Type + Name string +} + +func (h Handle) String() string { + name := h.Name + if len(name) > 10 { + name = name[:10] + } + return fmt.Sprintf("<%s/%s>", h.Type, name) +} + +// Valid returns an error if h is not valid. +func (h Handle) Valid() error { + if h.Type == "" { + return errors.New("type is empty") + } + + switch h.Type { + case Data: + case Key: + case Lock: + case Snapshot: + case Index: + case Config: + default: + return fmt.Errorf("invalid config %q", h.Type) + } + + if h.Type == Config { + return nil + } + + if h.Name == "" { + return errors.New("invalid Name") + } + + return nil +} diff --git a/backend/handle_test.go b/backend/handle_test.go new file mode 100644 index 000000000..a477c0aec --- /dev/null +++ b/backend/handle_test.go @@ -0,0 +1,28 @@ +package backend + +import "testing" + +var handleTests = []struct { + h Handle + valid bool +}{ + {Handle{Name: "foo"}, false}, + {Handle{Type: "foobar"}, false}, + {Handle{Type: Config, Name: ""}, true}, + {Handle{Type: Data, Name: ""}, false}, + {Handle{Type: "", Name: "x"}, false}, + {Handle{Type: Lock, Name: "010203040506"}, true}, +} + +func TestHandleValid(t *testing.T) { + for i, test := range handleTests { + err := test.h.Valid() + if err != nil && test.valid { + t.Errorf("test %v failed: error returned for valid handle: %v", i, err) + } + + if !test.valid && err == nil { + t.Errorf("test %v failed: expected error for invalid handle not found", i) + } + } +} diff --git a/backend/interface.go b/backend/interface.go index 73c3e82b6..1fdacb4d7 100644 --- a/backend/interface.go +++ b/backend/interface.go @@ -1,9 +1,6 @@ package backend -import ( - "fmt" - "io" -) +import "io" // Type is the type of a Blob. type Type string @@ -18,20 +15,6 @@ const ( Config = "config" ) -// Handle is used to store and access data in a backend. -type Handle struct { - Type Type - Name string -} - -func (h Handle) String() string { - name := h.Name - if len(name) > 10 { - name = name[:10] - } - return fmt.Sprintf("<%s/%s>", h.Type, name) -} - // Backend is used to store and access data. type Backend interface { // Location returns a string that describes the type and location of the diff --git a/backend/local/backend_test.go b/backend/local/backend_test.go new file mode 100644 index 000000000..8dea61892 --- /dev/null +++ b/backend/local/backend_test.go @@ -0,0 +1,20 @@ +// DO NOT EDIT! +// generated at 2016-23-01 17:06:38 +0100 CET +package local_test + +import ( + "testing" + + "github.com/restic/restic/backend/test" +) + +func TestCreate(t *testing.T) { test.Create(t) } +func TestOpen(t *testing.T) { test.Open(t) } +func TestLocation(t *testing.T) { test.Location(t) } +func TestConfig(t *testing.T) { test.Config(t) } +func TestGetReader(t *testing.T) { test.GetReader(t) } +func TestLoad(t *testing.T) { test.Load(t) } +func TestWrite(t *testing.T) { test.Write(t) } +func TestGeneric(t *testing.T) { test.Generic(t) } +func TestDelete(t *testing.T) { test.Delete(t) } +func TestCleanup(t *testing.T) { test.Cleanup(t) } diff --git a/backend/local/local.go b/backend/local/local.go index 9946ddc4a..25ba26779 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -226,6 +226,10 @@ func (b *Local) GetReader(t backend.Type, name string, offset, length uint) (io. // Load returns the data stored in the backend for h at the given offset // and saves it in p. Load has the same semantics as io.ReaderAt. func (b *Local) Load(h backend.Handle, p []byte, off int64) (n int, err error) { + if err := h.Valid(); err != nil { + return 0, err + } + f, err := os.Open(filename(b.p, h.Type, h.Name)) if err != nil { return 0, err diff --git a/backend/local/local_test.go b/backend/local/local_test.go new file mode 100644 index 000000000..d574b015c --- /dev/null +++ b/backend/local/local_test.go @@ -0,0 +1,53 @@ +package local_test + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + + "github.com/restic/restic/backend" + "github.com/restic/restic/backend/local" + "github.com/restic/restic/backend/test" +) + +var tempBackendDir string + +//go:generate go run ../test/generate_backend_tests.go + +func init() { + test.CreateFn = func() (backend.Backend, error) { + if tempBackendDir != "" { + return nil, errors.New("temporary local backend dir already exists") + } + + tempdir, err := ioutil.TempDir("", "restic-local-test-") + if err != nil { + return nil, err + } + + fmt.Printf("created new test backend at %v\n", tempdir) + tempBackendDir = tempdir + + return local.Create(tempdir) + } + + test.OpenFn = func() (backend.Backend, error) { + if tempBackendDir == "" { + return nil, errors.New("repository not initialized") + } + + return local.Open(tempBackendDir) + } + + test.CleanupFn = func() error { + if tempBackendDir == "" { + return nil + } + + fmt.Printf("removing test backend at %v\n", tempBackendDir) + err := os.RemoveAll(tempBackendDir) + tempBackendDir = "" + return err + } +} diff --git a/backend/test/generate_backend_tests.go b/backend/test/generate_backend_tests.go new file mode 100644 index 000000000..64c89159d --- /dev/null +++ b/backend/test/generate_backend_tests.go @@ -0,0 +1,111 @@ +// +build ignore + +package main + +import ( + "bufio" + "flag" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "regexp" + "text/template" + "time" +) + +var data struct { + Package string + Funcs []string + Timestamp string +} + +var testTemplate = ` +// DO NOT EDIT! +// generated at {{ .Timestamp }} +package {{ .Package }} + +import ( + "testing" + + "github.com/restic/restic/backend/test" +) + +{{ range $f := .Funcs }}func Test{{ $f }}(t *testing.T){ test.{{ $f }}(t) } +{{ end }} +` + +var testFile = flag.String("testfile", "../test/tests.go", "file to search test functions in") +var outputFile = flag.String("output", "backend_test.go", "output file to write generated code to") + +func errx(err error) { + if err == nil { + return + } + + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) +} + +var funcRegex = regexp.MustCompile(`^func\s+([A-Z].*)\s*\(`) + +func findTestFunctions() (funcs []string) { + f, err := os.Open(*testFile) + errx(err) + + sc := bufio.NewScanner(f) + for sc.Scan() { + match := funcRegex.FindStringSubmatch(sc.Text()) + if len(match) > 0 { + funcs = append(funcs, match[1]) + } + } + + if err := sc.Err(); err != nil { + log.Fatalf("Error scanning file: %v", err) + } + + errx(f.Close()) + return funcs +} + +func generateOutput(wr io.Writer, data interface{}) { + t := template.Must(template.New("backendtest").Parse(testTemplate)) + + cmd := exec.Command("gofmt") + cmd.Stdout = wr + in, err := cmd.StdinPipe() + errx(err) + errx(cmd.Start()) + errx(t.Execute(in, data)) + errx(in.Close()) + errx(cmd.Wait()) +} + +func init() { + flag.Parse() +} + +func main() { + dir, err := os.Getwd() + if err != nil { + fmt.Fprintf(os.Stderr, "Getwd() %v\n", err) + os.Exit(1) + } + + packageName := filepath.Base(dir) + + f, err := os.Create(*outputFile) + errx(err) + + data.Package = packageName + "_test" + data.Funcs = findTestFunctions() + data.Timestamp = time.Now().Format("2006-02-01 15:04:05 -0700 MST") + generateOutput(f, data) + + errx(f.Close()) + + fmt.Printf("wrote backend tests for package %v\n", packageName) +} diff --git a/backend/test/tests.go b/backend/test/tests.go new file mode 100644 index 000000000..65a3e723a --- /dev/null +++ b/backend/test/tests.go @@ -0,0 +1,550 @@ +package test + +import ( + "bytes" + crand "crypto/rand" + "fmt" + "io" + "io/ioutil" + "math/rand" + "reflect" + "sort" + "testing" + + "github.com/restic/restic/backend" + . "github.com/restic/restic/test" +) + +// CreateFn is a function that creates a temporary repository for the tests. +var CreateFn func() (backend.Backend, error) + +// OpenFn is a function that opens a previously created temporary repository. +var OpenFn func() (backend.Backend, error) + +// CleanupFn removes temporary files and directories created during the tests. +var CleanupFn func() error + +var but backend.Backend // backendUnderTest +var butInitialized bool + +func open(t testing.TB) backend.Backend { + if OpenFn == nil { + t.Fatal("OpenFn not set") + } + + if CreateFn == nil { + t.Fatalf("CreateFn not set") + } + + if !butInitialized { + be, err := CreateFn() + if err != nil { + t.Fatalf("Create returned unexpected error: %v", err) + } + + but = be + butInitialized = true + } + + if but == nil { + var err error + but, err = OpenFn() + if err != nil { + t.Fatalf("Open returned unexpected error: %v", err) + } + } + + return but +} + +func close(t testing.TB) { + if but == nil { + t.Fatalf("trying to close non-existing backend") + } + + err := but.Close() + if err != nil { + t.Fatalf("Close returned unexpected error: %v", err) + } + + but = nil +} + +// Create creates a backend. +func Create(t testing.TB) { + if CreateFn == nil { + t.Fatalf("CreateFn not set!") + } + + be, err := CreateFn() + if err != nil { + fmt.Printf("foo\n") + t.Fatalf("Create returned error: %v", err) + } + + butInitialized = true + + err = be.Close() + if err != nil { + t.Fatalf("Close returned error: %v", err) + } +} + +// Open opens a previously created backend. +func Open(t testing.TB) { + if OpenFn == nil { + t.Fatalf("OpenFn not set!") + } + + be, err := OpenFn() + if err != nil { + t.Fatalf("Open returned error: %v", err) + } + + err = be.Close() + if err != nil { + t.Fatalf("Close returned error: %v", err) + } +} + +// Location tests that a location string is returned. +func Location(t testing.TB) { + b := open(t) + defer close(t) + + l := b.Location() + if l == "" { + t.Fatalf("invalid location string %q", l) + } +} + +// Config saves and loads a config from the backend. +func Config(t testing.TB) { + b := open(t) + defer close(t) + + var testString = "Config" + + // create config and read it back + _, err := b.GetReader(backend.Config, "", 0, 0) + if err == nil { + t.Fatalf("did not get expected error for non-existing config") + } + + blob, err := b.Create() + if err != nil { + t.Fatalf("Create() error: %v", err) + } + + _, err = blob.Write([]byte(testString)) + if err != nil { + t.Fatalf("Write() error: %v", err) + } + + err = blob.Finalize(backend.Config, "") + if err != nil { + t.Fatalf("Finalize() error: %v", err) + } + + // try accessing the config with different names, should all return the + // same config + for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} { + rd, err := b.GetReader(backend.Config, name, 0, 0) + if err != nil { + t.Fatalf("unable to read config with name %q: %v", name, err) + } + + buf, err := ioutil.ReadAll(rd) + if err != nil { + t.Fatalf("read config error: %v", err) + } + + err = rd.Close() + if err != nil { + t.Fatalf("close error: %v", err) + } + + if string(buf) != testString { + t.Fatalf("wrong data returned, want %q, got %q", testString, string(buf)) + } + } +} + +// GetReader tests various ways the GetReader function can be called. +func GetReader(t testing.TB) { + b := open(t) + defer close(t) + + length := rand.Intn(1<<24) + 2000 + + data := make([]byte, length) + _, err := io.ReadFull(crand.Reader, data) + OK(t, err) + + blob, err := b.Create() + OK(t, err) + + id := backend.Hash(data) + + _, err = blob.Write([]byte(data)) + OK(t, err) + OK(t, blob.Finalize(backend.Data, id.String())) + + for i := 0; i < 500; i++ { + l := rand.Intn(length + 2000) + o := rand.Intn(length + 2000) + + d := data + if o < len(d) { + d = d[o:] + } else { + o = len(d) + d = d[:0] + } + + if l > 0 && l < len(d) { + d = d[:l] + } + + rd, err := b.GetReader(backend.Data, id.String(), uint(o), uint(l)) + OK(t, err) + buf, err := ioutil.ReadAll(rd) + OK(t, err) + + if !bytes.Equal(buf, d) { + t.Fatalf("data not equal") + } + } + + OK(t, b.Remove(backend.Data, id.String())) +} + +// Load tests the backend's Load function. +func Load(t testing.TB) { + b := open(t) + defer close(t) + + _, err := b.Load(backend.Handle{}, nil, 0) + if err == nil { + t.Fatalf("Load() did not return an error for invalid handle") + } + + _, err = b.Load(backend.Handle{Type: backend.Data, Name: "foobar"}, nil, 0) + if err == nil { + t.Fatalf("Load() did not return an error for non-existing blob") + } + + length := rand.Intn(1<<24) + 2000 + + data := make([]byte, length) + _, err = io.ReadFull(crand.Reader, data) + if err != nil { + t.Fatalf("reading random data failed: %v", err) + } + + id := backend.Hash(data) + + blob, err := b.Create() + OK(t, err) + + _, err = blob.Write([]byte(data)) + OK(t, err) + OK(t, blob.Finalize(backend.Data, id.String())) + + for i := 0; i < 500; i++ { + l := rand.Intn(length + 2000) + o := rand.Intn(length + 2000) + + d := data + if o < len(d) { + d = d[o:] + } else { + o = len(d) + d = d[:0] + } + + if l > 0 && l < len(d) { + d = d[:l] + } + + buf := make([]byte, l) + h := backend.Handle{Type: backend.Data, Name: id.String()} + n, err := b.Load(h, buf, int64(o)) + + // if we requested data beyond the end of the file, ignore + // ErrUnexpectedEOF error + if l > len(d) && err == io.ErrUnexpectedEOF { + err = nil + buf = buf[:len(d)] + } + + if err != nil { + t.Errorf("Load(%d, %d): unexpected error: %v", len(buf), int64(o), err) + continue + } + + if n != len(buf) { + t.Errorf("Load(%d, %d): wrong length returned, want %d, got %d", + len(buf), int64(o), len(buf), n) + continue + } + + buf = buf[:n] + if !bytes.Equal(buf, d) { + t.Errorf("Load(%d, %d) returned wrong bytes", len(buf), int64(o)) + continue + } + } + + OK(t, b.Remove(backend.Data, id.String())) +} + +// Write tests writing data to the backend. +func Write(t testing.TB) { + b := open(t) + defer close(t) + + length := rand.Intn(1<<23) + 2000 + + data := make([]byte, length) + _, err := io.ReadFull(crand.Reader, data) + OK(t, err) + id := backend.Hash(data) + + for i := 0; i < 10; i++ { + blob, err := b.Create() + OK(t, err) + + o := 0 + for o < len(data) { + l := rand.Intn(len(data) - o) + if len(data)-o < 20 { + l = len(data) - o + } + + n, err := blob.Write(data[o : o+l]) + OK(t, err) + if n != l { + t.Fatalf("wrong number of bytes written, want %v, got %v", l, n) + } + + o += l + } + + name := fmt.Sprintf("%s-%d", id, i) + OK(t, blob.Finalize(backend.Data, name)) + + rd, err := b.GetReader(backend.Data, name, 0, 0) + OK(t, err) + + buf, err := ioutil.ReadAll(rd) + OK(t, err) + + if len(buf) != len(data) { + t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf)) + } + + if !bytes.Equal(buf, data) { + t.Fatalf("data not equal") + } + + err = b.Remove(backend.Data, name) + if err != nil { + t.Fatalf("error removing item: %v", err) + } + } +} + +var testStrings = []struct { + id string + data string +}{ + {"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", "foobar"}, + {"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + {"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", "foo/bar"}, + {"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"}, +} + +func store(t testing.TB, b backend.Backend, tpe backend.Type, data []byte) { + id := backend.Hash(data) + + blob, err := b.Create() + OK(t, err) + + _, err = blob.Write([]byte(data)) + OK(t, err) + OK(t, blob.Finalize(tpe, id.String())) +} + +func read(t testing.TB, rd io.Reader, expectedData []byte) { + buf, err := ioutil.ReadAll(rd) + OK(t, err) + if expectedData != nil { + Equals(t, expectedData, buf) + } +} + +// Generic tests all functions of the backend. +func Generic(t testing.TB) { + b := open(t) + defer close(t) + + for _, tpe := range []backend.Type{ + backend.Data, backend.Key, backend.Lock, + backend.Snapshot, backend.Index, + } { + // detect non-existing files + for _, test := range testStrings { + id, err := backend.ParseID(test.id) + OK(t, err) + + // test if blob is already in repository + ret, err := b.Test(tpe, id.String()) + OK(t, err) + Assert(t, !ret, "blob was found to exist before creating") + + // try to open not existing blob + _, err = b.GetReader(tpe, id.String(), 0, 0) + Assert(t, err != nil, "blob data could be extracted before creation") + + // try to read not existing blob + _, err = b.GetReader(tpe, id.String(), 0, 1) + Assert(t, err != nil, "blob reader could be obtained before creation") + + // try to get string out, should fail + ret, err = b.Test(tpe, id.String()) + OK(t, err) + Assert(t, !ret, "id %q was found (but should not have)", test.id) + } + + // add files + for _, test := range testStrings { + store(t, b, tpe, []byte(test.data)) + + // test GetReader() + rd, err := b.GetReader(tpe, test.id, 0, uint(len(test.data))) + OK(t, err) + Assert(t, rd != nil, "GetReader() returned nil") + + read(t, rd, []byte(test.data)) + OK(t, rd.Close()) + + // try to read it out with an offset and a length + start := 1 + end := len(test.data) - 2 + length := end - start + rd, err = b.GetReader(tpe, test.id, uint(start), uint(length)) + OK(t, err) + Assert(t, rd != nil, "GetReader() returned nil") + + read(t, rd, []byte(test.data[start:end])) + OK(t, rd.Close()) + } + + // test adding the first file again + test := testStrings[0] + + // create blob + blob, err := b.Create() + OK(t, err) + + _, err = blob.Write([]byte(test.data)) + OK(t, err) + err = blob.Finalize(tpe, test.id) + Assert(t, err != nil, "expected error, got %v", err) + + // remove and recreate + err = b.Remove(tpe, test.id) + OK(t, err) + + // test that the blob is gone + ok, err := b.Test(tpe, test.id) + OK(t, err) + Assert(t, ok == false, "removed blob still present") + + // create blob + blob, err = b.Create() + OK(t, err) + + _, err = io.Copy(blob, bytes.NewReader([]byte(test.data))) + OK(t, err) + OK(t, blob.Finalize(tpe, test.id)) + + // list items + IDs := backend.IDs{} + + for _, test := range testStrings { + id, err := backend.ParseID(test.id) + OK(t, err) + IDs = append(IDs, id) + } + + list := backend.IDs{} + + for s := range b.List(tpe, nil) { + list = append(list, ParseID(s)) + } + + if len(IDs) != len(list) { + t.Fatalf("wrong number of IDs returned: want %d, got %d", len(IDs), len(list)) + } + + sort.Sort(IDs) + sort.Sort(list) + + if !reflect.DeepEqual(IDs, list) { + t.Fatalf("lists aren't equal, want:\n %v\n got:\n%v\n", IDs, list) + } + + // remove content if requested + if TestCleanup { + for _, test := range testStrings { + id, err := backend.ParseID(test.id) + OK(t, err) + + found, err := b.Test(tpe, id.String()) + OK(t, err) + + OK(t, b.Remove(tpe, id.String())) + + found, err = b.Test(tpe, id.String()) + OK(t, err) + Assert(t, !found, fmt.Sprintf("id %q not found after removal", id)) + } + } + } +} + +// Delete tests the Delete function. +func Delete(t testing.TB) { + b := open(t) + defer close(t) + + be, ok := b.(backend.Deleter) + if !ok { + return + } + + err := be.Delete() + if err != nil { + t.Fatalf("error deleting backend: %v", err) + } +} + +// Cleanup runs the cleanup function after all tests are run. +func Cleanup(t testing.TB) { + if CleanupFn == nil { + t.Log("CleanupFn function not set") + return + } + + if !TestCleanup { + t.Logf("not cleaning up backend") + return + } + + err := CleanupFn() + if err != nil { + t.Fatalf("Cleanup returned error: %v", err) + } +} diff --git a/backend/test/tests_test.go b/backend/test/tests_test.go new file mode 100644 index 000000000..fbdba9534 --- /dev/null +++ b/backend/test/tests_test.go @@ -0,0 +1,3 @@ +package test + +// func Test From 3aafa21887aac7572fae8af71d624ee05176db83 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 17:19:47 +0100 Subject: [PATCH 06/56] Fix MockBackend.Load() --- backend/mock_backend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/mock_backend.go b/backend/mock_backend.go index a5e21b8a4..744cb7902 100644 --- a/backend/mock_backend.go +++ b/backend/mock_backend.go @@ -48,7 +48,7 @@ func (m *MockBackend) Load(h Handle, p []byte, off int64) (int, error) { return 0, errors.New("not implemented") } - return m.Load(h, p, off) + return m.LoadFn(h, p, off) } func (m *MockBackend) GetReader(t Type, name string, offset, len uint) (io.ReadCloser, error) { From e966df3fed8f6300eda72d6632214249098eaaa3 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 17:19:55 +0100 Subject: [PATCH 07/56] Add Load() to MemBackend --- backend/mem_backend.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/backend/mem_backend.go b/backend/mem_backend.go index e689584c2..e4782e40c 100644 --- a/backend/mem_backend.go +++ b/backend/mem_backend.go @@ -45,6 +45,10 @@ func NewMemoryBackend() *MemoryBackend { return memGetReader(be, t, name, offset, length) } + be.MockBackend.LoadFn = func(h Handle, p []byte, off int64) (int, error) { + return memLoad(be, h, p, off) + } + be.MockBackend.RemoveFn = func(t Type, name string) error { return memRemove(be, t, name) } @@ -61,6 +65,10 @@ func NewMemoryBackend() *MemoryBackend { return nil } + be.MockBackend.LocationFn = func() string { + return "Memory Backend" + } + debug.Log("MemoryBackend.New", "created new memory backend") return be @@ -171,6 +179,40 @@ func memGetReader(be *MemoryBackend, t Type, name string, offset, length uint) ( return readCloser{bytes.NewReader(buf)}, nil } +func memLoad(be *MemoryBackend, h Handle, p []byte, off int64) (int, error) { + be.m.Lock() + defer be.m.Unlock() + + if err := h.Valid(); err != nil { + return 0, err + } + + if h.Type == Config { + h.Name = "" + } + + debug.Log("MemoryBackend.Load", "get %v offset %v len %v", h, off, len(p)) + + if _, ok := be.data[entry{h.Type, h.Name}]; !ok { + return 0, errors.New("no such data") + } + + buf := be.data[entry{h.Type, h.Name}] + if off > int64(len(buf)) { + return 0, errors.New("offset beyond end of file") + } + + buf = buf[off:] + + n := copy(p, buf) + + if len(p) > len(buf) { + return n, io.ErrUnexpectedEOF + } + + return n, nil +} + func memRemove(be *MemoryBackend, t Type, name string) error { be.m.Lock() defer be.m.Unlock() From 942376782772711ecca6da5c66ce2c747b410247 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 17:42:26 +0100 Subject: [PATCH 08/56] Update test generate script, add tests to membackend --- backend/local/backend_test.go | 22 +++++++-------- backend/mem_backend_prepare_test.go | 37 +++++++++++++++++++++++++ backend/mem_backend_test.go | 18 ++++++++---- backend/test/backend_test.go | 20 ++++++++++++++ backend/test/generate_backend_tests.go | 37 ++++++++++++++++++++----- backend/test/tests_test.go | 38 ++++++++++++++++++++++++-- 6 files changed, 147 insertions(+), 25 deletions(-) create mode 100644 backend/mem_backend_prepare_test.go create mode 100644 backend/test/backend_test.go diff --git a/backend/local/backend_test.go b/backend/local/backend_test.go index 8dea61892..8f521bced 100644 --- a/backend/local/backend_test.go +++ b/backend/local/backend_test.go @@ -1,5 +1,5 @@ // DO NOT EDIT! -// generated at 2016-23-01 17:06:38 +0100 CET +// generated at 2016-23-01 17:41:09 +0100 CET package local_test import ( @@ -8,13 +8,13 @@ import ( "github.com/restic/restic/backend/test" ) -func TestCreate(t *testing.T) { test.Create(t) } -func TestOpen(t *testing.T) { test.Open(t) } -func TestLocation(t *testing.T) { test.Location(t) } -func TestConfig(t *testing.T) { test.Config(t) } -func TestGetReader(t *testing.T) { test.GetReader(t) } -func TestLoad(t *testing.T) { test.Load(t) } -func TestWrite(t *testing.T) { test.Write(t) } -func TestGeneric(t *testing.T) { test.Generic(t) } -func TestDelete(t *testing.T) { test.Delete(t) } -func TestCleanup(t *testing.T) { test.Cleanup(t) } +func TestLocalBackendCreate(t *testing.T) { test.Create(t) } +func TestLocalBackendOpen(t *testing.T) { test.Open(t) } +func TestLocalBackendLocation(t *testing.T) { test.Location(t) } +func TestLocalBackendConfig(t *testing.T) { test.Config(t) } +func TestLocalBackendGetReader(t *testing.T) { test.GetReader(t) } +func TestLocalBackendLoad(t *testing.T) { test.Load(t) } +func TestLocalBackendWrite(t *testing.T) { test.Write(t) } +func TestLocalBackendGeneric(t *testing.T) { test.Generic(t) } +func TestLocalBackendDelete(t *testing.T) { test.Delete(t) } +func TestLocalBackendCleanup(t *testing.T) { test.Cleanup(t) } diff --git a/backend/mem_backend_prepare_test.go b/backend/mem_backend_prepare_test.go new file mode 100644 index 000000000..8ddeaa72c --- /dev/null +++ b/backend/mem_backend_prepare_test.go @@ -0,0 +1,37 @@ +package backend_test + +import ( + "errors" + + "github.com/restic/restic/backend" + "github.com/restic/restic/backend/test" +) + +var be backend.Backend + +//go:generate go run test/generate_backend_tests.go -testfile test/tests.go -output mem_backend_test.go -prefix MemBackend + +func init() { + test.CreateFn = func() (backend.Backend, error) { + if be != nil { + return nil, errors.New("temporary memory backend dir already exists") + } + + be = backend.NewMemoryBackend() + + return be, nil + } + + test.OpenFn = func() (backend.Backend, error) { + if be == nil { + return nil, errors.New("repository not initialized") + } + + return be, nil + } + + test.CleanupFn = func() error { + be = nil + return nil + } +} diff --git a/backend/mem_backend_test.go b/backend/mem_backend_test.go index c5b43415c..83e869990 100644 --- a/backend/mem_backend_test.go +++ b/backend/mem_backend_test.go @@ -1,12 +1,20 @@ +// DO NOT EDIT! +// generated at 2016-23-01 17:41:47 +0100 CET package backend_test import ( "testing" - "github.com/restic/restic/backend" + "github.com/restic/restic/backend/test" ) -func TestMemoryBackend(t *testing.T) { - be := backend.NewMemoryBackend() - testBackend(be, t) -} +func TestMemBackendCreate(t *testing.T) { test.Create(t) } +func TestMemBackendOpen(t *testing.T) { test.Open(t) } +func TestMemBackendLocation(t *testing.T) { test.Location(t) } +func TestMemBackendConfig(t *testing.T) { test.Config(t) } +func TestMemBackendGetReader(t *testing.T) { test.GetReader(t) } +func TestMemBackendLoad(t *testing.T) { test.Load(t) } +func TestMemBackendWrite(t *testing.T) { test.Write(t) } +func TestMemBackendGeneric(t *testing.T) { test.Generic(t) } +func TestMemBackendDelete(t *testing.T) { test.Delete(t) } +func TestMemBackendCleanup(t *testing.T) { test.Cleanup(t) } diff --git a/backend/test/backend_test.go b/backend/test/backend_test.go new file mode 100644 index 000000000..4c11a5ac3 --- /dev/null +++ b/backend/test/backend_test.go @@ -0,0 +1,20 @@ +// DO NOT EDIT! +// generated at 2016-23-01 17:41:27 +0100 CET +package test_test + +import ( + "testing" + + "github.com/restic/restic/backend/test" +) + +func TestTestBackendCreate(t *testing.T) { test.Create(t) } +func TestTestBackendOpen(t *testing.T) { test.Open(t) } +func TestTestBackendLocation(t *testing.T) { test.Location(t) } +func TestTestBackendConfig(t *testing.T) { test.Config(t) } +func TestTestBackendGetReader(t *testing.T) { test.GetReader(t) } +func TestTestBackendLoad(t *testing.T) { test.Load(t) } +func TestTestBackendWrite(t *testing.T) { test.Write(t) } +func TestTestBackendGeneric(t *testing.T) { test.Generic(t) } +func TestTestBackendDelete(t *testing.T) { test.Delete(t) } +func TestTestBackendCleanup(t *testing.T) { test.Cleanup(t) } diff --git a/backend/test/generate_backend_tests.go b/backend/test/generate_backend_tests.go index 64c89159d..2286ba2dd 100644 --- a/backend/test/generate_backend_tests.go +++ b/backend/test/generate_backend_tests.go @@ -14,12 +14,15 @@ import ( "regexp" "text/template" "time" + "unicode" + "unicode/utf8" ) var data struct { - Package string - Funcs []string - Timestamp string + Package string + PackagePrefix string + Funcs []string + Timestamp string } var testTemplate = ` @@ -33,12 +36,15 @@ import ( "github.com/restic/restic/backend/test" ) -{{ range $f := .Funcs }}func Test{{ $f }}(t *testing.T){ test.{{ $f }}(t) } +{{ $prefix := .PackagePrefix }} +{{ range $f := .Funcs }}func Test{{ $prefix }}{{ $f }}(t *testing.T){ test.{{ $f }}(t) } {{ end }} ` var testFile = flag.String("testfile", "../test/tests.go", "file to search test functions in") var outputFile = flag.String("output", "backend_test.go", "output file to write generated code to") +var packageName = flag.String("package", "", "the package name to use") +var prefix = flag.String("prefix", "", "test function prefix") func errx(err error) { if err == nil { @@ -84,6 +90,15 @@ func generateOutput(wr io.Writer, data interface{}) { errx(cmd.Wait()) } +func packageTestFunctionPrefix(pkg string) string { + if pkg == "" { + return "" + } + + r, n := utf8.DecodeRuneInString(pkg) + return string(unicode.ToUpper(r)) + pkg[n:] +} + func init() { flag.Parse() } @@ -95,17 +110,25 @@ func main() { os.Exit(1) } - packageName := filepath.Base(dir) + pkg := *packageName + if pkg == "" { + pkg = filepath.Base(dir) + } f, err := os.Create(*outputFile) errx(err) - data.Package = packageName + "_test" + data.Package = pkg + "_test" + if *prefix != "" { + data.PackagePrefix = *prefix + } else { + data.PackagePrefix = packageTestFunctionPrefix(pkg) + "Backend" + } data.Funcs = findTestFunctions() data.Timestamp = time.Now().Format("2006-02-01 15:04:05 -0700 MST") generateOutput(f, data) errx(f.Close()) - fmt.Printf("wrote backend tests for package %v\n", packageName) + fmt.Printf("wrote backend tests for package %v to %v\n", data.Package, *outputFile) } diff --git a/backend/test/tests_test.go b/backend/test/tests_test.go index fbdba9534..c901e6658 100644 --- a/backend/test/tests_test.go +++ b/backend/test/tests_test.go @@ -1,3 +1,37 @@ -package test +package test_test -// func Test +import ( + "errors" + + "github.com/restic/restic/backend" + "github.com/restic/restic/backend/test" +) + +var be backend.Backend + +//go:generate go run ../test/generate_backend_tests.go + +func init() { + test.CreateFn = func() (backend.Backend, error) { + if be != nil { + return nil, errors.New("temporary memory backend dir already exists") + } + + be = backend.NewMemoryBackend() + + return be, nil + } + + test.OpenFn = func() (backend.Backend, error) { + if be == nil { + return nil, errors.New("repository not initialized") + } + + return be, nil + } + + test.CleanupFn = func() error { + be = nil + return nil + } +} From 99fab793c076ecc580c5807a05dd3c67a38e19f4 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 17:43:49 +0100 Subject: [PATCH 09/56] Remove timestamp from generated tests --- backend/local/backend_test.go | 3 +-- backend/mem_backend_test.go | 3 +-- backend/test/backend_test.go | 3 +-- backend/test/generate_backend_tests.go | 6 +----- 4 files changed, 4 insertions(+), 11 deletions(-) diff --git a/backend/local/backend_test.go b/backend/local/backend_test.go index 8f521bced..14e865df5 100644 --- a/backend/local/backend_test.go +++ b/backend/local/backend_test.go @@ -1,5 +1,4 @@ -// DO NOT EDIT! -// generated at 2016-23-01 17:41:09 +0100 CET +// DO NOT EDIT, AUTOMATICALLY GENERATED package local_test import ( diff --git a/backend/mem_backend_test.go b/backend/mem_backend_test.go index 83e869990..0e04e97f2 100644 --- a/backend/mem_backend_test.go +++ b/backend/mem_backend_test.go @@ -1,5 +1,4 @@ -// DO NOT EDIT! -// generated at 2016-23-01 17:41:47 +0100 CET +// DO NOT EDIT, AUTOMATICALLY GENERATED package backend_test import ( diff --git a/backend/test/backend_test.go b/backend/test/backend_test.go index 4c11a5ac3..3af250808 100644 --- a/backend/test/backend_test.go +++ b/backend/test/backend_test.go @@ -1,5 +1,4 @@ -// DO NOT EDIT! -// generated at 2016-23-01 17:41:27 +0100 CET +// DO NOT EDIT, AUTOMATICALLY GENERATED package test_test import ( diff --git a/backend/test/generate_backend_tests.go b/backend/test/generate_backend_tests.go index 2286ba2dd..a08250129 100644 --- a/backend/test/generate_backend_tests.go +++ b/backend/test/generate_backend_tests.go @@ -13,7 +13,6 @@ import ( "path/filepath" "regexp" "text/template" - "time" "unicode" "unicode/utf8" ) @@ -22,12 +21,10 @@ var data struct { Package string PackagePrefix string Funcs []string - Timestamp string } var testTemplate = ` -// DO NOT EDIT! -// generated at {{ .Timestamp }} +// DO NOT EDIT, AUTOMATICALLY GENERATED package {{ .Package }} import ( @@ -125,7 +122,6 @@ func main() { data.PackagePrefix = packageTestFunctionPrefix(pkg) + "Backend" } data.Funcs = findTestFunctions() - data.Timestamp = time.Now().Format("2006-02-01 15:04:05 -0700 MST") generateOutput(f, data) errx(f.Close()) From 16b7cc7655db2323addfbf1dbce8800cd03f6c58 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 17:45:33 +0100 Subject: [PATCH 10/56] Remove redundant local tests --- backend/local_test.go | 59 ------------------------------------------- 1 file changed, 59 deletions(-) delete mode 100644 backend/local_test.go diff --git a/backend/local_test.go b/backend/local_test.go deleted file mode 100644 index 462c4c3d6..000000000 --- a/backend/local_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package backend_test - -import ( - "fmt" - "io/ioutil" - "testing" - - "github.com/restic/restic/backend" - "github.com/restic/restic/backend/local" - . "github.com/restic/restic/test" -) - -func setupLocalBackend(t *testing.T) *local.Local { - tempdir, err := ioutil.TempDir("", "restic-test-") - OK(t, err) - - b, err := local.Create(tempdir) - OK(t, err) - - t.Logf("created local backend at %s", tempdir) - - return b -} - -func teardownLocalBackend(t *testing.T, b *local.Local) { - if !TestCleanup { - t.Logf("leaving local backend at %s\n", b.Location()) - return - } - - OK(t, b.Delete()) -} - -func TestLocalBackend(t *testing.T) { - // test for non-existing backend - b, err := local.Open("/invalid-restic-test") - Assert(t, err != nil, "opening invalid repository at /invalid-restic-test should have failed, but err is nil") - Assert(t, b == nil, fmt.Sprintf("opening invalid repository at /invalid-restic-test should have failed, but b is not nil: %v", b)) - - s := setupLocalBackend(t) - defer teardownLocalBackend(t, s) - - testBackend(s, t) -} - -func TestLocalBackendCreationFailures(t *testing.T) { - b := setupLocalBackend(t) - defer teardownLocalBackend(t, b) - - // create a fake config file - blob, err := b.Create() - OK(t, err) - fmt.Fprintf(blob, "config\n") - OK(t, blob.Finalize(backend.Config, "")) - - // test failure to create a new repository at the same location - b2, err := local.Create(b.Location()) - Assert(t, err != nil && b2 == nil, fmt.Sprintf("creating a repository at %s for the second time should have failed", b.Location())) -} From 4952f8668251e6ac3c74cbf02652b7f797e07313 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 18:07:15 +0100 Subject: [PATCH 11/56] Add test for to prevent double create --- backend/local/backend_test.go | 21 +++++++++++---------- backend/local/local_test.go | 34 ++++++++++++++++++++-------------- backend/mem_backend_test.go | 21 +++++++++++---------- backend/test/backend_test.go | 21 +++++++++++---------- backend/test/tests.go | 26 ++++++++++++++++++++++++++ 5 files changed, 79 insertions(+), 44 deletions(-) diff --git a/backend/local/backend_test.go b/backend/local/backend_test.go index 14e865df5..2745f6e2c 100644 --- a/backend/local/backend_test.go +++ b/backend/local/backend_test.go @@ -7,13 +7,14 @@ import ( "github.com/restic/restic/backend/test" ) -func TestLocalBackendCreate(t *testing.T) { test.Create(t) } -func TestLocalBackendOpen(t *testing.T) { test.Open(t) } -func TestLocalBackendLocation(t *testing.T) { test.Location(t) } -func TestLocalBackendConfig(t *testing.T) { test.Config(t) } -func TestLocalBackendGetReader(t *testing.T) { test.GetReader(t) } -func TestLocalBackendLoad(t *testing.T) { test.Load(t) } -func TestLocalBackendWrite(t *testing.T) { test.Write(t) } -func TestLocalBackendGeneric(t *testing.T) { test.Generic(t) } -func TestLocalBackendDelete(t *testing.T) { test.Delete(t) } -func TestLocalBackendCleanup(t *testing.T) { test.Cleanup(t) } +func TestLocalBackendCreate(t *testing.T) { test.Create(t) } +func TestLocalBackendOpen(t *testing.T) { test.Open(t) } +func TestLocalBackendCreateWithConfig(t *testing.T) { test.CreateWithConfig(t) } +func TestLocalBackendLocation(t *testing.T) { test.Location(t) } +func TestLocalBackendConfig(t *testing.T) { test.Config(t) } +func TestLocalBackendGetReader(t *testing.T) { test.GetReader(t) } +func TestLocalBackendLoad(t *testing.T) { test.Load(t) } +func TestLocalBackendWrite(t *testing.T) { test.Write(t) } +func TestLocalBackendGeneric(t *testing.T) { test.Generic(t) } +func TestLocalBackendDelete(t *testing.T) { test.Delete(t) } +func TestLocalBackendCleanup(t *testing.T) { test.Cleanup(t) } diff --git a/backend/local/local_test.go b/backend/local/local_test.go index d574b015c..3335cbfa8 100644 --- a/backend/local/local_test.go +++ b/backend/local/local_test.go @@ -1,7 +1,6 @@ package local_test import ( - "errors" "fmt" "io/ioutil" "os" @@ -15,28 +14,35 @@ var tempBackendDir string //go:generate go run ../test/generate_backend_tests.go +func createTempdir() error { + if tempBackendDir != "" { + return nil + } + + tempdir, err := ioutil.TempDir("", "restic-local-test-") + if err != nil { + return err + } + + fmt.Printf("created new test backend at %v\n", tempdir) + tempBackendDir = tempdir + return nil +} + func init() { test.CreateFn = func() (backend.Backend, error) { - if tempBackendDir != "" { - return nil, errors.New("temporary local backend dir already exists") - } - - tempdir, err := ioutil.TempDir("", "restic-local-test-") + err := createTempdir() if err != nil { return nil, err } - - fmt.Printf("created new test backend at %v\n", tempdir) - tempBackendDir = tempdir - - return local.Create(tempdir) + return local.Create(tempBackendDir) } test.OpenFn = func() (backend.Backend, error) { - if tempBackendDir == "" { - return nil, errors.New("repository not initialized") + err := createTempdir() + if err != nil { + return nil, err } - return local.Open(tempBackendDir) } diff --git a/backend/mem_backend_test.go b/backend/mem_backend_test.go index 0e04e97f2..73a012e74 100644 --- a/backend/mem_backend_test.go +++ b/backend/mem_backend_test.go @@ -7,13 +7,14 @@ import ( "github.com/restic/restic/backend/test" ) -func TestMemBackendCreate(t *testing.T) { test.Create(t) } -func TestMemBackendOpen(t *testing.T) { test.Open(t) } -func TestMemBackendLocation(t *testing.T) { test.Location(t) } -func TestMemBackendConfig(t *testing.T) { test.Config(t) } -func TestMemBackendGetReader(t *testing.T) { test.GetReader(t) } -func TestMemBackendLoad(t *testing.T) { test.Load(t) } -func TestMemBackendWrite(t *testing.T) { test.Write(t) } -func TestMemBackendGeneric(t *testing.T) { test.Generic(t) } -func TestMemBackendDelete(t *testing.T) { test.Delete(t) } -func TestMemBackendCleanup(t *testing.T) { test.Cleanup(t) } +func TestMemBackendCreate(t *testing.T) { test.Create(t) } +func TestMemBackendOpen(t *testing.T) { test.Open(t) } +func TestMemBackendCreateWithConfig(t *testing.T) { test.CreateWithConfig(t) } +func TestMemBackendLocation(t *testing.T) { test.Location(t) } +func TestMemBackendConfig(t *testing.T) { test.Config(t) } +func TestMemBackendGetReader(t *testing.T) { test.GetReader(t) } +func TestMemBackendLoad(t *testing.T) { test.Load(t) } +func TestMemBackendWrite(t *testing.T) { test.Write(t) } +func TestMemBackendGeneric(t *testing.T) { test.Generic(t) } +func TestMemBackendDelete(t *testing.T) { test.Delete(t) } +func TestMemBackendCleanup(t *testing.T) { test.Cleanup(t) } diff --git a/backend/test/backend_test.go b/backend/test/backend_test.go index 3af250808..db6de1b5f 100644 --- a/backend/test/backend_test.go +++ b/backend/test/backend_test.go @@ -7,13 +7,14 @@ import ( "github.com/restic/restic/backend/test" ) -func TestTestBackendCreate(t *testing.T) { test.Create(t) } -func TestTestBackendOpen(t *testing.T) { test.Open(t) } -func TestTestBackendLocation(t *testing.T) { test.Location(t) } -func TestTestBackendConfig(t *testing.T) { test.Config(t) } -func TestTestBackendGetReader(t *testing.T) { test.GetReader(t) } -func TestTestBackendLoad(t *testing.T) { test.Load(t) } -func TestTestBackendWrite(t *testing.T) { test.Write(t) } -func TestTestBackendGeneric(t *testing.T) { test.Generic(t) } -func TestTestBackendDelete(t *testing.T) { test.Delete(t) } -func TestTestBackendCleanup(t *testing.T) { test.Cleanup(t) } +func TestTestBackendCreate(t *testing.T) { test.Create(t) } +func TestTestBackendOpen(t *testing.T) { test.Open(t) } +func TestTestBackendCreateWithConfig(t *testing.T) { test.CreateWithConfig(t) } +func TestTestBackendLocation(t *testing.T) { test.Location(t) } +func TestTestBackendConfig(t *testing.T) { test.Config(t) } +func TestTestBackendGetReader(t *testing.T) { test.GetReader(t) } +func TestTestBackendLoad(t *testing.T) { test.Load(t) } +func TestTestBackendWrite(t *testing.T) { test.Write(t) } +func TestTestBackendGeneric(t *testing.T) { test.Generic(t) } +func TestTestBackendDelete(t *testing.T) { test.Delete(t) } +func TestTestBackendCleanup(t *testing.T) { test.Cleanup(t) } diff --git a/backend/test/tests.go b/backend/test/tests.go index 65a3e723a..f99b5a60c 100644 --- a/backend/test/tests.go +++ b/backend/test/tests.go @@ -107,6 +107,32 @@ func Open(t testing.TB) { } } +// CreateWithConfig tests that creating a backend in a location which already +// has a config file fails. +func CreateWithConfig(t testing.TB) { + if CreateFn == nil { + t.Fatalf("CreateFn not set") + } + + b := open(t) + defer close(t) + + // save a config + store(t, b, backend.Config, []byte("test config")) + + // now create the backend again, this must fail + _, err := CreateFn() + if err == nil { + t.Fatalf("expected error not found for creating a backend with an existing config file") + } + + // remove config + err = b.Remove(backend.Config, "") + if err != nil { + t.Fatalf("unexpected error removing config: %v", err) + } +} + // Location tests that a location string is returned. func Location(t testing.TB) { b := open(t) From c6db567e3fde05a46d70b35199472ffe4fea6309 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 18:30:02 +0100 Subject: [PATCH 12/56] Add sftp tests --- backend/local/backend_test.go | 89 ++++++++++++++++++++++---- backend/mem_backend_test.go | 89 ++++++++++++++++++++++---- backend/sftp/backend_test.go | 87 +++++++++++++++++++++++++ backend/sftp/sftp.go | 4 ++ backend/sftp/sftp_backend_test.go | 81 +++++++++++++++++++++++ backend/sftp_test.go | 65 ------------------- backend/test/backend_test.go | 89 ++++++++++++++++++++++---- backend/test/generate_backend_tests.go | 14 +++- 8 files changed, 418 insertions(+), 100 deletions(-) create mode 100644 backend/sftp/backend_test.go create mode 100644 backend/sftp/sftp_backend_test.go delete mode 100644 backend/sftp_test.go diff --git a/backend/local/backend_test.go b/backend/local/backend_test.go index 2745f6e2c..e1ac22dd8 100644 --- a/backend/local/backend_test.go +++ b/backend/local/backend_test.go @@ -7,14 +7,81 @@ import ( "github.com/restic/restic/backend/test" ) -func TestLocalBackendCreate(t *testing.T) { test.Create(t) } -func TestLocalBackendOpen(t *testing.T) { test.Open(t) } -func TestLocalBackendCreateWithConfig(t *testing.T) { test.CreateWithConfig(t) } -func TestLocalBackendLocation(t *testing.T) { test.Location(t) } -func TestLocalBackendConfig(t *testing.T) { test.Config(t) } -func TestLocalBackendGetReader(t *testing.T) { test.GetReader(t) } -func TestLocalBackendLoad(t *testing.T) { test.Load(t) } -func TestLocalBackendWrite(t *testing.T) { test.Write(t) } -func TestLocalBackendGeneric(t *testing.T) { test.Generic(t) } -func TestLocalBackendDelete(t *testing.T) { test.Delete(t) } -func TestLocalBackendCleanup(t *testing.T) { test.Cleanup(t) } +var SkipMessage string + +func TestLocalBackendCreate(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Create(t) +} + +func TestLocalBackendOpen(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Open(t) +} + +func TestLocalBackendCreateWithConfig(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.CreateWithConfig(t) +} + +func TestLocalBackendLocation(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Location(t) +} + +func TestLocalBackendConfig(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Config(t) +} + +func TestLocalBackendGetReader(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.GetReader(t) +} + +func TestLocalBackendLoad(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Load(t) +} + +func TestLocalBackendWrite(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Write(t) +} + +func TestLocalBackendGeneric(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Generic(t) +} + +func TestLocalBackendDelete(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Delete(t) +} + +func TestLocalBackendCleanup(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Cleanup(t) +} diff --git a/backend/mem_backend_test.go b/backend/mem_backend_test.go index 73a012e74..41ce60041 100644 --- a/backend/mem_backend_test.go +++ b/backend/mem_backend_test.go @@ -7,14 +7,81 @@ import ( "github.com/restic/restic/backend/test" ) -func TestMemBackendCreate(t *testing.T) { test.Create(t) } -func TestMemBackendOpen(t *testing.T) { test.Open(t) } -func TestMemBackendCreateWithConfig(t *testing.T) { test.CreateWithConfig(t) } -func TestMemBackendLocation(t *testing.T) { test.Location(t) } -func TestMemBackendConfig(t *testing.T) { test.Config(t) } -func TestMemBackendGetReader(t *testing.T) { test.GetReader(t) } -func TestMemBackendLoad(t *testing.T) { test.Load(t) } -func TestMemBackendWrite(t *testing.T) { test.Write(t) } -func TestMemBackendGeneric(t *testing.T) { test.Generic(t) } -func TestMemBackendDelete(t *testing.T) { test.Delete(t) } -func TestMemBackendCleanup(t *testing.T) { test.Cleanup(t) } +var SkipMessage string + +func TestMemBackendCreate(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Create(t) +} + +func TestMemBackendOpen(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Open(t) +} + +func TestMemBackendCreateWithConfig(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.CreateWithConfig(t) +} + +func TestMemBackendLocation(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Location(t) +} + +func TestMemBackendConfig(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Config(t) +} + +func TestMemBackendGetReader(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.GetReader(t) +} + +func TestMemBackendLoad(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Load(t) +} + +func TestMemBackendWrite(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Write(t) +} + +func TestMemBackendGeneric(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Generic(t) +} + +func TestMemBackendDelete(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Delete(t) +} + +func TestMemBackendCleanup(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Cleanup(t) +} diff --git a/backend/sftp/backend_test.go b/backend/sftp/backend_test.go new file mode 100644 index 000000000..973c7e4aa --- /dev/null +++ b/backend/sftp/backend_test.go @@ -0,0 +1,87 @@ +// DO NOT EDIT, AUTOMATICALLY GENERATED +package sftp_test + +import ( + "testing" + + "github.com/restic/restic/backend/test" +) + +var SkipMessage string + +func TestSftpBackendCreate(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Create(t) +} + +func TestSftpBackendOpen(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Open(t) +} + +func TestSftpBackendCreateWithConfig(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.CreateWithConfig(t) +} + +func TestSftpBackendLocation(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Location(t) +} + +func TestSftpBackendConfig(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Config(t) +} + +func TestSftpBackendGetReader(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.GetReader(t) +} + +func TestSftpBackendLoad(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Load(t) +} + +func TestSftpBackendWrite(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Write(t) +} + +func TestSftpBackendGeneric(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Generic(t) +} + +func TestSftpBackendDelete(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Delete(t) +} + +func TestSftpBackendCleanup(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Cleanup(t) +} diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 007e2088a..a3d32e3d6 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -369,6 +369,10 @@ func (r *SFTP) GetReader(t backend.Type, name string, offset, length uint) (io.R // Load returns the data stored in the backend for h at the given offset // and saves it in p. Load has the same semantics as io.ReaderAt. func (r *SFTP) Load(h backend.Handle, p []byte, off int64) (n int, err error) { + if err := h.Valid(); err != nil { + return 0, err + } + f, err := r.c.Open(r.filename(h.Type, h.Name)) if err != nil { return 0, err diff --git a/backend/sftp/sftp_backend_test.go b/backend/sftp/sftp_backend_test.go new file mode 100644 index 000000000..431abff2a --- /dev/null +++ b/backend/sftp/sftp_backend_test.go @@ -0,0 +1,81 @@ +package sftp_test + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/restic/restic/backend" + "github.com/restic/restic/backend/sftp" + "github.com/restic/restic/backend/test" + + . "github.com/restic/restic/test" +) + +var tempBackendDir string + +//go:generate go run ../test/generate_backend_tests.go + +func createTempdir() error { + if tempBackendDir != "" { + return nil + } + + tempdir, err := ioutil.TempDir("", "restic-local-test-") + if err != nil { + return err + } + + fmt.Printf("created new test backend at %v\n", tempdir) + tempBackendDir = tempdir + return nil +} + +func init() { + sftpserver := "" + + for _, dir := range strings.Split(TestSFTPPath, ":") { + testpath := filepath.Join(dir, "sftp-server") + fd, err := os.Open(testpath) + fd.Close() + if !os.IsNotExist(err) { + sftpserver = testpath + break + } + } + + if sftpserver == "" { + SkipMessage = "sftp server binary not found, skipping tests" + return + } + + test.CreateFn = func() (backend.Backend, error) { + err := createTempdir() + if err != nil { + return nil, err + } + + return sftp.Create(tempBackendDir, sftpserver) + } + + test.OpenFn = func() (backend.Backend, error) { + err := createTempdir() + if err != nil { + return nil, err + } + return sftp.Open(tempBackendDir, sftpserver) + } + + test.CleanupFn = func() error { + if tempBackendDir == "" { + return nil + } + + fmt.Printf("removing test backend at %v\n", tempBackendDir) + err := os.RemoveAll(tempBackendDir) + tempBackendDir = "" + return err + } +} diff --git a/backend/sftp_test.go b/backend/sftp_test.go deleted file mode 100644 index b678e8ea9..000000000 --- a/backend/sftp_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package backend_test - -import ( - "io/ioutil" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/restic/restic/backend/sftp" - . "github.com/restic/restic/test" -) - -func setupSFTPBackend(t *testing.T) *sftp.SFTP { - sftpserver := "" - - for _, dir := range strings.Split(TestSFTPPath, ":") { - testpath := filepath.Join(dir, "sftp-server") - fd, err := os.Open(testpath) - fd.Close() - if !os.IsNotExist(err) { - sftpserver = testpath - break - } - } - - if sftpserver == "" { - return nil - } - - tempdir, err := ioutil.TempDir("", "restic-test-") - OK(t, err) - - b, err := sftp.Create(tempdir, sftpserver) - OK(t, err) - - t.Logf("created sftp backend locally at %s", tempdir) - - return b -} - -func teardownSFTPBackend(t *testing.T, b *sftp.SFTP) { - if !TestCleanup { - t.Logf("leaving backend at %s\n", b.Location()) - return - } - - err := os.RemoveAll(b.Location()) - OK(t, err) -} - -func TestSFTPBackend(t *testing.T) { - if !RunIntegrationTest { - t.Skip("integration tests disabled") - } - - s := setupSFTPBackend(t) - if s == nil { - t.Skip("unable to find sftp-server binary") - return - } - defer teardownSFTPBackend(t, s) - - testBackend(s, t) -} diff --git a/backend/test/backend_test.go b/backend/test/backend_test.go index db6de1b5f..f53fb1e47 100644 --- a/backend/test/backend_test.go +++ b/backend/test/backend_test.go @@ -7,14 +7,81 @@ import ( "github.com/restic/restic/backend/test" ) -func TestTestBackendCreate(t *testing.T) { test.Create(t) } -func TestTestBackendOpen(t *testing.T) { test.Open(t) } -func TestTestBackendCreateWithConfig(t *testing.T) { test.CreateWithConfig(t) } -func TestTestBackendLocation(t *testing.T) { test.Location(t) } -func TestTestBackendConfig(t *testing.T) { test.Config(t) } -func TestTestBackendGetReader(t *testing.T) { test.GetReader(t) } -func TestTestBackendLoad(t *testing.T) { test.Load(t) } -func TestTestBackendWrite(t *testing.T) { test.Write(t) } -func TestTestBackendGeneric(t *testing.T) { test.Generic(t) } -func TestTestBackendDelete(t *testing.T) { test.Delete(t) } -func TestTestBackendCleanup(t *testing.T) { test.Cleanup(t) } +var SkipMessage string + +func TestTestBackendCreate(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Create(t) +} + +func TestTestBackendOpen(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Open(t) +} + +func TestTestBackendCreateWithConfig(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.CreateWithConfig(t) +} + +func TestTestBackendLocation(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Location(t) +} + +func TestTestBackendConfig(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Config(t) +} + +func TestTestBackendGetReader(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.GetReader(t) +} + +func TestTestBackendLoad(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Load(t) +} + +func TestTestBackendWrite(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Write(t) +} + +func TestTestBackendGeneric(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Generic(t) +} + +func TestTestBackendDelete(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Delete(t) +} + +func TestTestBackendCleanup(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Cleanup(t) +} diff --git a/backend/test/generate_backend_tests.go b/backend/test/generate_backend_tests.go index a08250129..9a0aa2968 100644 --- a/backend/test/generate_backend_tests.go +++ b/backend/test/generate_backend_tests.go @@ -33,8 +33,15 @@ import ( "github.com/restic/restic/backend/test" ) +var SkipMessage string + {{ $prefix := .PackagePrefix }} -{{ range $f := .Funcs }}func Test{{ $prefix }}{{ $f }}(t *testing.T){ test.{{ $f }}(t) } +{{ range $f := .Funcs }} +func Test{{ $prefix }}{{ $f }}(t *testing.T){ + if SkipMessage != "" { t.Skip(SkipMessage) } + test.{{ $f }}(t) +} + {{ end }} ` @@ -42,6 +49,7 @@ var testFile = flag.String("testfile", "../test/tests.go", "file to search test var outputFile = flag.String("output", "backend_test.go", "output file to write generated code to") var packageName = flag.String("package", "", "the package name to use") var prefix = flag.String("prefix", "", "test function prefix") +var quiet = flag.Bool("quiet", false, "be quiet") func errx(err error) { if err == nil { @@ -126,5 +134,7 @@ func main() { errx(f.Close()) - fmt.Printf("wrote backend tests for package %v to %v\n", data.Package, *outputFile) + if !*quiet { + fmt.Printf("wrote backend tests for package %v to %v\n", data.Package, *outputFile) + } } From 15c8b85a4ba35395725524c3c0c1175df44e0eba Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 18:46:04 +0100 Subject: [PATCH 13/56] Add tests for s3 backend --- backend/s3/backend_test.go | 87 ++++++++++++++++++++++++++++++++++++++ backend/s3/s3_test.go | 71 +++++++++++++++++++++++++++++-- 2 files changed, 155 insertions(+), 3 deletions(-) create mode 100644 backend/s3/backend_test.go diff --git a/backend/s3/backend_test.go b/backend/s3/backend_test.go new file mode 100644 index 000000000..f8f824bd6 --- /dev/null +++ b/backend/s3/backend_test.go @@ -0,0 +1,87 @@ +// DO NOT EDIT, AUTOMATICALLY GENERATED +package s3_test + +import ( + "testing" + + "github.com/restic/restic/backend/test" +) + +var SkipMessage string + +func TestS3BackendCreate(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Create(t) +} + +func TestS3BackendOpen(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Open(t) +} + +func TestS3BackendCreateWithConfig(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.CreateWithConfig(t) +} + +func TestS3BackendLocation(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Location(t) +} + +func TestS3BackendConfig(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Config(t) +} + +func TestS3BackendGetReader(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.GetReader(t) +} + +func TestS3BackendLoad(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Load(t) +} + +func TestS3BackendWrite(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Write(t) +} + +func TestS3BackendGeneric(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Generic(t) +} + +func TestS3BackendDelete(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Delete(t) +} + +func TestS3BackendCleanup(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.Cleanup(t) +} diff --git a/backend/s3/s3_test.go b/backend/s3/s3_test.go index 289748485..050e4300a 100644 --- a/backend/s3/s3_test.go +++ b/backend/s3/s3_test.go @@ -1,7 +1,72 @@ -package s3 +package s3_test -import "testing" +import ( + "errors" + "fmt" + "net/url" + "os" -func TestGetReader(t *testing.T) { + "github.com/restic/restic/backend" + "github.com/restic/restic/backend/s3" + "github.com/restic/restic/backend/test" + . "github.com/restic/restic/test" +) +//go:generate go run ../test/generate_backend_tests.go + +func init() { + if TestS3Server == "" { + SkipMessage = "s3 test server not available" + return + } + + url, err := url.Parse(TestS3Server) + if err != nil { + fmt.Fprintf(os.Stderr, "invalid url: %v\n", err) + return + } + + cfg := s3.Config{ + Endpoint: url.Host, + Bucket: "restictestbucket", + KeyID: os.Getenv("AWS_ACCESS_KEY_ID"), + Secret: os.Getenv("AWS_SECRET_ACCESS_KEY"), + } + + if url.Scheme == "http" { + cfg.UseHTTP = true + } + + test.CreateFn = func() (backend.Backend, error) { + be, err := s3.Open(cfg) + if err != nil { + return nil, err + } + + exists, err := be.Test(backend.Config, "") + if err != nil { + return nil, err + } + + if exists { + return nil, errors.New("config already exists") + } + + return be, nil + } + + test.OpenFn = func() (backend.Backend, error) { + return s3.Open(cfg) + } + + // test.CleanupFn = func() error { + // if tempBackendDir == "" { + // return nil + // } + + // fmt.Printf("removing test backend at %v\n", tempBackendDir) + // err := os.RemoveAll(tempBackendDir) + // tempBackendDir = "" + // return err + // } } From 6ba56befad0ba68fca1be106388a776a9d02e9b1 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 19:10:43 +0100 Subject: [PATCH 14/56] Abort fuse integration test on error Before, the fuse integration test was run and the tests were never finished, because the testing code did not detect any errors when the fusermount binary returned an error. This commit fixes it. --- cmd/restic/integration_fuse_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/restic/integration_fuse_test.go b/cmd/restic/integration_fuse_test.go index 807b335fc..5c5c2021f 100644 --- a/cmd/restic/integration_fuse_test.go +++ b/cmd/restic/integration_fuse_test.go @@ -54,6 +54,10 @@ func waitForMount(dir string) error { } func cmdMount(t testing.TB, global GlobalOptions, dir string, ready, done chan struct{}) { + defer func() { + ready <- struct{}{} + }() + cmd := &CmdMount{global: &global, ready: ready, done: done} OK(t, cmd.Execute([]string{dir})) if TestCleanup { @@ -104,7 +108,7 @@ func TestMount(t *testing.T) { // We remove the mountpoint now to check that cmdMount creates it RemoveAll(t, mountpoint) - ready := make(chan struct{}, 1) + ready := make(chan struct{}, 2) done := make(chan struct{}) go cmdMount(t, global, mountpoint, ready, done) <-ready From e4f2e4a2035c81e19d12dd9171c45d6749982836 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 19:11:47 +0100 Subject: [PATCH 15/56] Remove old s3 tests --- backend/backend_test.go | 275 ---------------------------------------- backend/s3_test.go | 42 ------ 2 files changed, 317 deletions(-) delete mode 100644 backend/backend_test.go delete mode 100644 backend/s3_test.go diff --git a/backend/backend_test.go b/backend/backend_test.go deleted file mode 100644 index 297170f56..000000000 --- a/backend/backend_test.go +++ /dev/null @@ -1,275 +0,0 @@ -package backend_test - -import ( - "bytes" - "fmt" - "io" - "io/ioutil" - "math/rand" - "sort" - "testing" - - crand "crypto/rand" - - "github.com/restic/restic/backend" - . "github.com/restic/restic/test" -) - -func testBackendConfig(b backend.Backend, t *testing.T) { - // create config and read it back - _, err := b.GetReader(backend.Config, "", 0, 0) - Assert(t, err != nil, "did not get expected error for non-existing config") - - blob, err := b.Create() - OK(t, err) - - _, err = blob.Write([]byte("Config")) - OK(t, err) - OK(t, blob.Finalize(backend.Config, "")) - - // try accessing the config with different names, should all return the - // same config - for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} { - rd, err := b.GetReader(backend.Config, name, 0, 0) - Assert(t, err == nil, "unable to read config") - - buf, err := ioutil.ReadAll(rd) - OK(t, err) - OK(t, rd.Close()) - Assert(t, string(buf) == "Config", "wrong data returned for config") - } -} - -func testGetReader(b backend.Backend, t testing.TB) { - length := rand.Intn(1<<24) + 2000 - - data := make([]byte, length) - _, err := io.ReadFull(crand.Reader, data) - OK(t, err) - - blob, err := b.Create() - OK(t, err) - - id := backend.Hash(data) - - _, err = blob.Write([]byte(data)) - OK(t, err) - OK(t, blob.Finalize(backend.Data, id.String())) - - for i := 0; i < 500; i++ { - l := rand.Intn(length + 2000) - o := rand.Intn(length + 2000) - - d := data - if o < len(d) { - d = d[o:] - } else { - o = len(d) - d = d[:0] - } - - if l > 0 && l < len(d) { - d = d[:l] - } - - rd, err := b.GetReader(backend.Data, id.String(), uint(o), uint(l)) - OK(t, err) - buf, err := ioutil.ReadAll(rd) - OK(t, err) - - if !bytes.Equal(buf, d) { - t.Fatalf("data not equal") - } - } - - OK(t, b.Remove(backend.Data, id.String())) -} - -func testWrite(b backend.Backend, t testing.TB) { - length := rand.Intn(1<<23) + 2000 - - data := make([]byte, length) - _, err := io.ReadFull(crand.Reader, data) - OK(t, err) - id := backend.Hash(data) - - for i := 0; i < 10; i++ { - blob, err := b.Create() - OK(t, err) - - o := 0 - for o < len(data) { - l := rand.Intn(len(data) - o) - if len(data)-o < 20 { - l = len(data) - o - } - - n, err := blob.Write(data[o : o+l]) - OK(t, err) - if n != l { - t.Fatalf("wrong number of bytes written, want %v, got %v", l, n) - } - - o += l - } - - name := fmt.Sprintf("%s-%d", id, i) - OK(t, blob.Finalize(backend.Data, name)) - - rd, err := b.GetReader(backend.Data, name, 0, 0) - OK(t, err) - - buf, err := ioutil.ReadAll(rd) - OK(t, err) - - if len(buf) != len(data) { - t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf)) - } - - if !bytes.Equal(buf, data) { - t.Fatalf("data not equal") - } - } -} - -func store(t testing.TB, b backend.Backend, tpe backend.Type, data []byte) { - id := backend.Hash(data) - - blob, err := b.Create() - OK(t, err) - - _, err = blob.Write([]byte(data)) - OK(t, err) - OK(t, blob.Finalize(tpe, id.String())) -} - -func read(t testing.TB, rd io.Reader, expectedData []byte) { - buf, err := ioutil.ReadAll(rd) - OK(t, err) - if expectedData != nil { - Equals(t, expectedData, buf) - } -} - -func testBackend(b backend.Backend, t *testing.T) { - testBackendConfig(b, t) - - for _, tpe := range []backend.Type{ - backend.Data, backend.Key, backend.Lock, - backend.Snapshot, backend.Index, - } { - // detect non-existing files - for _, test := range TestStrings { - id, err := backend.ParseID(test.id) - OK(t, err) - - // test if blob is already in repository - ret, err := b.Test(tpe, id.String()) - OK(t, err) - Assert(t, !ret, "blob was found to exist before creating") - - // try to open not existing blob - _, err = b.GetReader(tpe, id.String(), 0, 0) - Assert(t, err != nil, "blob data could be extracted before creation") - - // try to read not existing blob - _, err = b.GetReader(tpe, id.String(), 0, 1) - Assert(t, err != nil, "blob reader could be obtained before creation") - - // try to get string out, should fail - ret, err = b.Test(tpe, id.String()) - OK(t, err) - Assert(t, !ret, "id %q was found (but should not have)", test.id) - } - - // add files - for _, test := range TestStrings { - store(t, b, tpe, []byte(test.data)) - - // test GetReader() - rd, err := b.GetReader(tpe, test.id, 0, uint(len(test.data))) - OK(t, err) - Assert(t, rd != nil, "GetReader() returned nil") - - read(t, rd, []byte(test.data)) - OK(t, rd.Close()) - - // try to read it out with an offset and a length - start := 1 - end := len(test.data) - 2 - length := end - start - rd, err = b.GetReader(tpe, test.id, uint(start), uint(length)) - OK(t, err) - Assert(t, rd != nil, "GetReader() returned nil") - - read(t, rd, []byte(test.data[start:end])) - OK(t, rd.Close()) - } - - // test adding the first file again - test := TestStrings[0] - - // create blob - blob, err := b.Create() - OK(t, err) - - _, err = blob.Write([]byte(test.data)) - OK(t, err) - err = blob.Finalize(tpe, test.id) - Assert(t, err != nil, "expected error, got %v", err) - - // remove and recreate - err = b.Remove(tpe, test.id) - OK(t, err) - - // test that the blob is gone - ok, err := b.Test(tpe, test.id) - OK(t, err) - Assert(t, ok == false, "removed blob still present") - - // create blob - blob, err = b.Create() - OK(t, err) - - _, err = io.Copy(blob, bytes.NewReader([]byte(test.data))) - OK(t, err) - OK(t, blob.Finalize(tpe, test.id)) - - // list items - IDs := backend.IDs{} - - for _, test := range TestStrings { - id, err := backend.ParseID(test.id) - OK(t, err) - IDs = append(IDs, id) - } - - sort.Sort(IDs) - - i := 0 - for s := range b.List(tpe, nil) { - Equals(t, IDs[i].String(), s) - i++ - } - - // remove content if requested - if TestCleanup { - for _, test := range TestStrings { - id, err := backend.ParseID(test.id) - OK(t, err) - - found, err := b.Test(tpe, id.String()) - OK(t, err) - - OK(t, b.Remove(tpe, id.String())) - - found, err = b.Test(tpe, id.String()) - OK(t, err) - Assert(t, !found, fmt.Sprintf("id %q not found after removal", id)) - } - } - } - - testGetReader(b, t) - testWrite(b, t) -} diff --git a/backend/s3_test.go b/backend/s3_test.go deleted file mode 100644 index 6d79f9cd8..000000000 --- a/backend/s3_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package backend_test - -import ( - "net/url" - "os" - "testing" - - "github.com/restic/restic/backend/s3" - . "github.com/restic/restic/test" -) - -type deleter interface { - Delete() error -} - -func TestS3Backend(t *testing.T) { - if TestS3Server == "" { - t.Skip("s3 test server not available") - } - - url, err := url.Parse(TestS3Server) - OK(t, err) - - cfg := s3.Config{ - Endpoint: url.Host, - Bucket: "restictestbucket", - KeyID: os.Getenv("AWS_ACCESS_KEY_ID"), - Secret: os.Getenv("AWS_SECRET_ACCESS_KEY"), - } - - if url.Scheme == "http" { - cfg.UseHTTP = true - } - - be, err := s3.Open(cfg) - OK(t, err) - - testBackend(be, t) - - del := be.(deleter) - OK(t, del.Delete()) -} From f05a32509e5535bce43047b7133afd1839162410 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 19:12:02 +0100 Subject: [PATCH 16/56] Add "Test" prefix to backend test functions --- backend/local/backend_test.go | 24 ++++++------- backend/mem_backend_test.go | 24 ++++++------- backend/s3/backend_test.go | 24 ++++++------- backend/sftp/backend_test.go | 24 ++++++------- backend/test/backend_test.go | 24 ++++++------- backend/test/generate_backend_tests.go | 4 +-- backend/test/tests.go | 48 +++++++++++++------------- cmd/restic/integration_fuse_test.go | 2 +- cmd/restic/integration_helpers_test.go | 4 +-- node_test.go | 2 +- test/backend.go | 4 +-- test/helpers.go | 2 +- tree_test.go | 2 +- 13 files changed, 94 insertions(+), 94 deletions(-) diff --git a/backend/local/backend_test.go b/backend/local/backend_test.go index e1ac22dd8..8cc244e85 100644 --- a/backend/local/backend_test.go +++ b/backend/local/backend_test.go @@ -13,75 +13,75 @@ func TestLocalBackendCreate(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Create(t) + test.TestCreate(t) } func TestLocalBackendOpen(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Open(t) + test.TestOpen(t) } func TestLocalBackendCreateWithConfig(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.CreateWithConfig(t) + test.TestCreateWithConfig(t) } func TestLocalBackendLocation(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Location(t) + test.TestLocation(t) } func TestLocalBackendConfig(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Config(t) + test.TestConfig(t) } func TestLocalBackendGetReader(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.GetReader(t) + test.TestGetReader(t) } func TestLocalBackendLoad(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Load(t) + test.TestLoad(t) } func TestLocalBackendWrite(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Write(t) + test.TestWrite(t) } -func TestLocalBackendGeneric(t *testing.T) { +func TestLocalBackendBackend(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Generic(t) + test.TestBackend(t) } func TestLocalBackendDelete(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Delete(t) + test.TestDelete(t) } func TestLocalBackendCleanup(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Cleanup(t) + test.TestCleanup(t) } diff --git a/backend/mem_backend_test.go b/backend/mem_backend_test.go index 41ce60041..46dfd88f4 100644 --- a/backend/mem_backend_test.go +++ b/backend/mem_backend_test.go @@ -13,75 +13,75 @@ func TestMemBackendCreate(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Create(t) + test.TestCreate(t) } func TestMemBackendOpen(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Open(t) + test.TestOpen(t) } func TestMemBackendCreateWithConfig(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.CreateWithConfig(t) + test.TestCreateWithConfig(t) } func TestMemBackendLocation(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Location(t) + test.TestLocation(t) } func TestMemBackendConfig(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Config(t) + test.TestConfig(t) } func TestMemBackendGetReader(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.GetReader(t) + test.TestGetReader(t) } func TestMemBackendLoad(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Load(t) + test.TestLoad(t) } func TestMemBackendWrite(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Write(t) + test.TestWrite(t) } -func TestMemBackendGeneric(t *testing.T) { +func TestMemBackendBackend(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Generic(t) + test.TestBackend(t) } func TestMemBackendDelete(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Delete(t) + test.TestDelete(t) } func TestMemBackendCleanup(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Cleanup(t) + test.TestCleanup(t) } diff --git a/backend/s3/backend_test.go b/backend/s3/backend_test.go index f8f824bd6..d0d15f51a 100644 --- a/backend/s3/backend_test.go +++ b/backend/s3/backend_test.go @@ -13,75 +13,75 @@ func TestS3BackendCreate(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Create(t) + test.TestCreate(t) } func TestS3BackendOpen(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Open(t) + test.TestOpen(t) } func TestS3BackendCreateWithConfig(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.CreateWithConfig(t) + test.TestCreateWithConfig(t) } func TestS3BackendLocation(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Location(t) + test.TestLocation(t) } func TestS3BackendConfig(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Config(t) + test.TestConfig(t) } func TestS3BackendGetReader(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.GetReader(t) + test.TestGetReader(t) } func TestS3BackendLoad(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Load(t) + test.TestLoad(t) } func TestS3BackendWrite(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Write(t) + test.TestWrite(t) } -func TestS3BackendGeneric(t *testing.T) { +func TestS3BackendBackend(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Generic(t) + test.TestBackend(t) } func TestS3BackendDelete(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Delete(t) + test.TestDelete(t) } func TestS3BackendCleanup(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Cleanup(t) + test.TestCleanup(t) } diff --git a/backend/sftp/backend_test.go b/backend/sftp/backend_test.go index 973c7e4aa..ef4ac9129 100644 --- a/backend/sftp/backend_test.go +++ b/backend/sftp/backend_test.go @@ -13,75 +13,75 @@ func TestSftpBackendCreate(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Create(t) + test.TestCreate(t) } func TestSftpBackendOpen(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Open(t) + test.TestOpen(t) } func TestSftpBackendCreateWithConfig(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.CreateWithConfig(t) + test.TestCreateWithConfig(t) } func TestSftpBackendLocation(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Location(t) + test.TestLocation(t) } func TestSftpBackendConfig(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Config(t) + test.TestConfig(t) } func TestSftpBackendGetReader(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.GetReader(t) + test.TestGetReader(t) } func TestSftpBackendLoad(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Load(t) + test.TestLoad(t) } func TestSftpBackendWrite(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Write(t) + test.TestWrite(t) } -func TestSftpBackendGeneric(t *testing.T) { +func TestSftpBackendBackend(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Generic(t) + test.TestBackend(t) } func TestSftpBackendDelete(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Delete(t) + test.TestDelete(t) } func TestSftpBackendCleanup(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Cleanup(t) + test.TestCleanup(t) } diff --git a/backend/test/backend_test.go b/backend/test/backend_test.go index f53fb1e47..44b330a0a 100644 --- a/backend/test/backend_test.go +++ b/backend/test/backend_test.go @@ -13,75 +13,75 @@ func TestTestBackendCreate(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Create(t) + test.TestCreate(t) } func TestTestBackendOpen(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Open(t) + test.TestOpen(t) } func TestTestBackendCreateWithConfig(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.CreateWithConfig(t) + test.TestCreateWithConfig(t) } func TestTestBackendLocation(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Location(t) + test.TestLocation(t) } func TestTestBackendConfig(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Config(t) + test.TestConfig(t) } func TestTestBackendGetReader(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.GetReader(t) + test.TestGetReader(t) } func TestTestBackendLoad(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Load(t) + test.TestLoad(t) } func TestTestBackendWrite(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Write(t) + test.TestWrite(t) } -func TestTestBackendGeneric(t *testing.T) { +func TestTestBackendBackend(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Generic(t) + test.TestBackend(t) } func TestTestBackendDelete(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Delete(t) + test.TestDelete(t) } func TestTestBackendCleanup(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) } - test.Cleanup(t) + test.TestCleanup(t) } diff --git a/backend/test/generate_backend_tests.go b/backend/test/generate_backend_tests.go index 9a0aa2968..0631f72c8 100644 --- a/backend/test/generate_backend_tests.go +++ b/backend/test/generate_backend_tests.go @@ -39,7 +39,7 @@ var SkipMessage string {{ range $f := .Funcs }} func Test{{ $prefix }}{{ $f }}(t *testing.T){ if SkipMessage != "" { t.Skip(SkipMessage) } - test.{{ $f }}(t) + test.Test{{ $f }}(t) } {{ end }} @@ -60,7 +60,7 @@ func errx(err error) { os.Exit(1) } -var funcRegex = regexp.MustCompile(`^func\s+([A-Z].*)\s*\(`) +var funcRegex = regexp.MustCompile(`^func\s+Test(.+)\s*\(`) func findTestFunctions() (funcs []string) { f, err := os.Open(*testFile) diff --git a/backend/test/tests.go b/backend/test/tests.go index f99b5a60c..2631fe168 100644 --- a/backend/test/tests.go +++ b/backend/test/tests.go @@ -70,8 +70,8 @@ func close(t testing.TB) { but = nil } -// Create creates a backend. -func Create(t testing.TB) { +// TestCreate creates a backend. +func TestCreate(t testing.TB) { if CreateFn == nil { t.Fatalf("CreateFn not set!") } @@ -90,8 +90,8 @@ func Create(t testing.TB) { } } -// Open opens a previously created backend. -func Open(t testing.TB) { +// TestOpen opens a previously created backend. +func TestOpen(t testing.TB) { if OpenFn == nil { t.Fatalf("OpenFn not set!") } @@ -107,9 +107,9 @@ func Open(t testing.TB) { } } -// CreateWithConfig tests that creating a backend in a location which already +// TestCreateWithConfig tests that creating a backend in a location which already // has a config file fails. -func CreateWithConfig(t testing.TB) { +func TestCreateWithConfig(t testing.TB) { if CreateFn == nil { t.Fatalf("CreateFn not set") } @@ -133,8 +133,8 @@ func CreateWithConfig(t testing.TB) { } } -// Location tests that a location string is returned. -func Location(t testing.TB) { +// TestLocation tests that a location string is returned. +func TestLocation(t testing.TB) { b := open(t) defer close(t) @@ -144,8 +144,8 @@ func Location(t testing.TB) { } } -// Config saves and loads a config from the backend. -func Config(t testing.TB) { +// TestConfig saves and loads a config from the backend. +func TestConfig(t testing.TB) { b := open(t) defer close(t) @@ -196,8 +196,8 @@ func Config(t testing.TB) { } } -// GetReader tests various ways the GetReader function can be called. -func GetReader(t testing.TB) { +// TestGetReader tests various ways the GetReader function can be called. +func TestGetReader(t testing.TB) { b := open(t) defer close(t) @@ -245,8 +245,8 @@ func GetReader(t testing.TB) { OK(t, b.Remove(backend.Data, id.String())) } -// Load tests the backend's Load function. -func Load(t testing.TB) { +// TestLoad tests the backend's Load function. +func TestLoad(t testing.TB) { b := open(t) defer close(t) @@ -325,8 +325,8 @@ func Load(t testing.TB) { OK(t, b.Remove(backend.Data, id.String())) } -// Write tests writing data to the backend. -func Write(t testing.TB) { +// TestWrite tests writing data to the backend. +func TestWrite(t testing.TB) { b := open(t) defer close(t) @@ -410,8 +410,8 @@ func read(t testing.TB, rd io.Reader, expectedData []byte) { } } -// Generic tests all functions of the backend. -func Generic(t testing.TB) { +// TestBackend tests all functions of the backend. +func TestBackend(t testing.TB) { b := open(t) defer close(t) @@ -523,7 +523,7 @@ func Generic(t testing.TB) { } // remove content if requested - if TestCleanup { + if TestCleanupTempDirs { for _, test := range testStrings { id, err := backend.ParseID(test.id) OK(t, err) @@ -541,8 +541,8 @@ func Generic(t testing.TB) { } } -// Delete tests the Delete function. -func Delete(t testing.TB) { +// TestDelete tests the Delete function. +func TestDelete(t testing.TB) { b := open(t) defer close(t) @@ -557,14 +557,14 @@ func Delete(t testing.TB) { } } -// Cleanup runs the cleanup function after all tests are run. -func Cleanup(t testing.TB) { +// TestCleanup runs the cleanup function after all tests are run. +func TestCleanup(t testing.TB) { if CleanupFn == nil { t.Log("CleanupFn function not set") return } - if !TestCleanup { + if !TestCleanupTempDirs { t.Logf("not cleaning up backend") return } diff --git a/cmd/restic/integration_fuse_test.go b/cmd/restic/integration_fuse_test.go index 5c5c2021f..1e696706b 100644 --- a/cmd/restic/integration_fuse_test.go +++ b/cmd/restic/integration_fuse_test.go @@ -60,7 +60,7 @@ func cmdMount(t testing.TB, global GlobalOptions, dir string, ready, done chan s cmd := &CmdMount{global: &global, ready: ready, done: done} OK(t, cmd.Execute([]string{dir})) - if TestCleanup { + if TestCleanupTempDirs { RemoveAll(t, dir) } } diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go index 734d974c1..b3bada889 100644 --- a/cmd/restic/integration_helpers_test.go +++ b/cmd/restic/integration_helpers_test.go @@ -178,7 +178,7 @@ func configureRestic(t testing.TB, cache, repo string) GlobalOptions { } func cleanupTempdir(t testing.TB, tempdir string) { - if !TestCleanup { + if !TestCleanupTempDirs { t.Logf("leaving temporary directory %v used for test", tempdir) return } @@ -209,7 +209,7 @@ func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions)) f(&env, configureRestic(t, env.cache, env.repo)) - if !TestCleanup { + if !TestCleanupTempDirs { t.Logf("leaving temporary directory %v used for test", tempdir) return } diff --git a/node_test.go b/node_test.go index ffb347e45..57bd6f550 100644 --- a/node_test.go +++ b/node_test.go @@ -145,7 +145,7 @@ func TestNodeRestoreAt(t *testing.T) { OK(t, err) defer func() { - if TestCleanup { + if TestCleanupTempDirs { RemoveAll(t, tempdir) } else { t.Logf("leaving tempdir at %v", tempdir) diff --git a/test/backend.go b/test/backend.go index 8dea18578..4da1f24f9 100644 --- a/test/backend.go +++ b/test/backend.go @@ -15,7 +15,7 @@ import ( var ( TestPassword = getStringVar("RESTIC_TEST_PASSWORD", "geheim") - TestCleanup = getBoolVar("RESTIC_TEST_CLEANUP", true) + TestCleanupTempDirs = getBoolVar("RESTIC_TEST_CLEANUP", true) TestTempDir = getStringVar("RESTIC_TEST_TMPDIR", "") RunIntegrationTest = getBoolVar("RESTIC_TEST_INTEGRATION", true) RunFuseTest = getBoolVar("RESTIC_TEST_FUSE", true) @@ -70,7 +70,7 @@ func SetupRepo() *repository.Repository { } func TeardownRepo(repo *repository.Repository) { - if !TestCleanup { + if !TestCleanupTempDirs { l := repo.Backend().(*local.Local) fmt.Printf("leaving local backend at %s\n", l.Location()) return diff --git a/test/helpers.go b/test/helpers.go index bfab04cc9..95a1f003c 100644 --- a/test/helpers.go +++ b/test/helpers.go @@ -158,7 +158,7 @@ func WithTestEnvironment(t testing.TB, repoFixture string, f func(repodir string f(filepath.Join(tempdir, "repo")) - if !TestCleanup { + if !TestCleanupTempDirs { t.Logf("leaving temporary directory %v used for test", tempdir) return } diff --git a/tree_test.go b/tree_test.go index 725e80b14..ad8f7f5fa 100644 --- a/tree_test.go +++ b/tree_test.go @@ -49,7 +49,7 @@ func createTempDir(t *testing.T) string { func TestTree(t *testing.T) { dir := createTempDir(t) defer func() { - if TestCleanup { + if TestCleanupTempDirs { RemoveAll(t, dir) } }() From 0b50f9e02cdea5cd4472cf5a9e545af0514b86b4 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 19:19:26 +0100 Subject: [PATCH 17/56] Move MemoryBackend to backend/mem --- .../backend_test.go} | 2 +- backend/{ => mem}/mem_backend.go | 66 +++++++------------ .../mem_backend_test.go} | 7 +- backend/readcloser.go | 21 ++++++ backend/test/tests_test.go | 3 +- checker/checker_test.go | 3 +- 6 files changed, 54 insertions(+), 48 deletions(-) rename backend/{mem_backend_test.go => mem/backend_test.go} (98%) rename backend/{ => mem}/mem_backend.go (67%) rename backend/{mem_backend_prepare_test.go => mem/mem_backend_test.go} (74%) create mode 100644 backend/readcloser.go diff --git a/backend/mem_backend_test.go b/backend/mem/backend_test.go similarity index 98% rename from backend/mem_backend_test.go rename to backend/mem/backend_test.go index 46dfd88f4..233b30f60 100644 --- a/backend/mem_backend_test.go +++ b/backend/mem/backend_test.go @@ -1,5 +1,5 @@ // DO NOT EDIT, AUTOMATICALLY GENERATED -package backend_test +package mem_test import ( "testing" diff --git a/backend/mem_backend.go b/backend/mem/mem_backend.go similarity index 67% rename from backend/mem_backend.go rename to backend/mem/mem_backend.go index e4782e40c..e231c2ede 100644 --- a/backend/mem_backend.go +++ b/backend/mem/mem_backend.go @@ -1,4 +1,4 @@ -package backend +package mem import ( "bytes" @@ -7,11 +7,12 @@ import ( "sort" "sync" + "github.com/restic/restic/backend" "github.com/restic/restic/debug" ) type entry struct { - Type Type + Type backend.Type Name string } @@ -23,37 +24,36 @@ type MemoryBackend struct { data memMap m sync.Mutex - MockBackend + backend.MockBackend } -// NewMemoryBackend returns a new backend that saves all data in a map in -// memory. -func NewMemoryBackend() *MemoryBackend { +// New returns a new backend that saves all data in a map in memory. +func New() *MemoryBackend { be := &MemoryBackend{ data: make(memMap), } - be.MockBackend.TestFn = func(t Type, name string) (bool, error) { + be.MockBackend.TestFn = func(t backend.Type, name string) (bool, error) { return memTest(be, t, name) } - be.MockBackend.CreateFn = func() (Blob, error) { + be.MockBackend.CreateFn = func() (backend.Blob, error) { return memCreate(be) } - be.MockBackend.GetReaderFn = func(t Type, name string, offset, length uint) (io.ReadCloser, error) { + be.MockBackend.GetReaderFn = func(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { return memGetReader(be, t, name, offset, length) } - be.MockBackend.LoadFn = func(h Handle, p []byte, off int64) (int, error) { + be.MockBackend.LoadFn = func(h backend.Handle, p []byte, off int64) (int, error) { return memLoad(be, h, p, off) } - be.MockBackend.RemoveFn = func(t Type, name string) error { + be.MockBackend.RemoveFn = func(t backend.Type, name string) error { return memRemove(be, t, name) } - be.MockBackend.ListFn = func(t Type, done <-chan struct{}) <-chan string { + be.MockBackend.ListFn = func(t backend.Type, done <-chan struct{}) <-chan string { return memList(be, t, done) } @@ -74,7 +74,7 @@ func NewMemoryBackend() *MemoryBackend { return be } -func (be *MemoryBackend) insert(t Type, name string, data []byte) error { +func (be *MemoryBackend) insert(t backend.Type, name string, data []byte) error { be.m.Lock() defer be.m.Unlock() @@ -86,7 +86,7 @@ func (be *MemoryBackend) insert(t Type, name string, data []byte) error { return nil } -func memTest(be *MemoryBackend, t Type, name string) (bool, error) { +func memTest(be *MemoryBackend, t backend.Type, name string) (bool, error) { be.m.Lock() defer be.m.Unlock() @@ -114,8 +114,8 @@ func (e *tempMemEntry) Size() uint { return uint(len(e.data.Bytes())) } -func (e *tempMemEntry) Finalize(t Type, name string) error { - if t == Config { +func (e *tempMemEntry) Finalize(t backend.Type, name string) error { + if t == backend.Config { name = "" } @@ -123,35 +123,17 @@ func (e *tempMemEntry) Finalize(t Type, name string) error { return e.be.insert(t, name, e.data.Bytes()) } -func memCreate(be *MemoryBackend) (Blob, error) { +func memCreate(be *MemoryBackend) (backend.Blob, error) { blob := &tempMemEntry{be: be} debug.Log("MemoryBackend.Create", "create new blob %p", blob) return blob, nil } -// ReadCloser wraps a reader and adds a noop Close method if rd does not implement io.Closer. -func ReadCloser(rd io.Reader) io.ReadCloser { - return readCloser{rd} -} - -// readCloser wraps a reader and adds a noop Close method if rd does not implement io.Closer. -type readCloser struct { - io.Reader -} - -func (rd readCloser) Close() error { - if r, ok := rd.Reader.(io.Closer); ok { - return r.Close() - } - - return nil -} - -func memGetReader(be *MemoryBackend, t Type, name string, offset, length uint) (io.ReadCloser, error) { +func memGetReader(be *MemoryBackend, t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { be.m.Lock() defer be.m.Unlock() - if t == Config { + if t == backend.Config { name = "" } @@ -176,10 +158,10 @@ func memGetReader(be *MemoryBackend, t Type, name string, offset, length uint) ( buf = buf[:length] } - return readCloser{bytes.NewReader(buf)}, nil + return backend.ReadCloser(bytes.NewReader(buf)), nil } -func memLoad(be *MemoryBackend, h Handle, p []byte, off int64) (int, error) { +func memLoad(be *MemoryBackend, h backend.Handle, p []byte, off int64) (int, error) { be.m.Lock() defer be.m.Unlock() @@ -187,7 +169,7 @@ func memLoad(be *MemoryBackend, h Handle, p []byte, off int64) (int, error) { return 0, err } - if h.Type == Config { + if h.Type == backend.Config { h.Name = "" } @@ -213,7 +195,7 @@ func memLoad(be *MemoryBackend, h Handle, p []byte, off int64) (int, error) { return n, nil } -func memRemove(be *MemoryBackend, t Type, name string) error { +func memRemove(be *MemoryBackend, t backend.Type, name string) error { be.m.Lock() defer be.m.Unlock() @@ -228,7 +210,7 @@ func memRemove(be *MemoryBackend, t Type, name string) error { return nil } -func memList(be *MemoryBackend, t Type, done <-chan struct{}) <-chan string { +func memList(be *MemoryBackend, t backend.Type, done <-chan struct{}) <-chan string { be.m.Lock() defer be.m.Unlock() diff --git a/backend/mem_backend_prepare_test.go b/backend/mem/mem_backend_test.go similarity index 74% rename from backend/mem_backend_prepare_test.go rename to backend/mem/mem_backend_test.go index 8ddeaa72c..8c3745aa2 100644 --- a/backend/mem_backend_prepare_test.go +++ b/backend/mem/mem_backend_test.go @@ -1,15 +1,16 @@ -package backend_test +package mem_test import ( "errors" "github.com/restic/restic/backend" + "github.com/restic/restic/backend/mem" "github.com/restic/restic/backend/test" ) var be backend.Backend -//go:generate go run test/generate_backend_tests.go -testfile test/tests.go -output mem_backend_test.go -prefix MemBackend +//go:generate go run ../test/generate_backend_tests.go func init() { test.CreateFn = func() (backend.Backend, error) { @@ -17,7 +18,7 @@ func init() { return nil, errors.New("temporary memory backend dir already exists") } - be = backend.NewMemoryBackend() + be = mem.New() return be, nil } diff --git a/backend/readcloser.go b/backend/readcloser.go new file mode 100644 index 000000000..6467e0dfc --- /dev/null +++ b/backend/readcloser.go @@ -0,0 +1,21 @@ +package backend + +import "io" + +// ReadCloser wraps a reader and adds a noop Close method if rd does not implement io.Closer. +func ReadCloser(rd io.Reader) io.ReadCloser { + return readCloser{rd} +} + +// readCloser wraps a reader and adds a noop Close method if rd does not implement io.Closer. +type readCloser struct { + io.Reader +} + +func (rd readCloser) Close() error { + if r, ok := rd.Reader.(io.Closer); ok { + return r.Close() + } + + return nil +} diff --git a/backend/test/tests_test.go b/backend/test/tests_test.go index c901e6658..22e769745 100644 --- a/backend/test/tests_test.go +++ b/backend/test/tests_test.go @@ -4,6 +4,7 @@ import ( "errors" "github.com/restic/restic/backend" + "github.com/restic/restic/backend/mem" "github.com/restic/restic/backend/test" ) @@ -17,7 +18,7 @@ func init() { return nil, errors.New("temporary memory backend dir already exists") } - be = backend.NewMemoryBackend() + be = mem.New() return be, nil } diff --git a/checker/checker_test.go b/checker/checker_test.go index 9ce2db5ca..6118b987d 100644 --- a/checker/checker_test.go +++ b/checker/checker_test.go @@ -9,6 +9,7 @@ import ( "github.com/restic/restic" "github.com/restic/restic/backend" + "github.com/restic/restic/backend/mem" "github.com/restic/restic/checker" "github.com/restic/restic/repository" . "github.com/restic/restic/test" @@ -252,7 +253,7 @@ func (f faultReader) Read(p []byte) (int, error) { } func TestCheckerModifiedData(t *testing.T) { - be := backend.NewMemoryBackend() + be := mem.New() repo := repository.New(be) OK(t, repo.Init(TestPassword)) From 10b03eee27a91bb72791fbc924855c7e09364e7b Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 23:27:40 +0100 Subject: [PATCH 18/56] Add comment --- repository/key.go | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/key.go b/repository/key.go index 5759849fa..ea8fbd4c7 100644 --- a/repository/key.go +++ b/repository/key.go @@ -225,6 +225,7 @@ func (k *Key) String() string { return fmt.Sprintf("", k.Username, k.Hostname, k.Created) } +// Name returns an identifier for the key. func (k Key) Name() string { return k.name } From 919b40c6cf1cfa919bfdf32b98008509a4687782 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 23:27:58 +0100 Subject: [PATCH 19/56] Add Stat() method to backend interface --- backend/interface.go | 8 ++++++++ backend/local/local.go | 14 ++++++++++++++ backend/mem/mem_backend.go | 26 ++++++++++++++++++++++++++ backend/mock_backend.go | 9 +++++++++ backend/s3/s3.go | 19 +++++++++++++++++++ backend/sftp/sftp.go | 14 ++++++++++++++ backend/test/tests.go | 7 +++++++ 7 files changed, 97 insertions(+) diff --git a/backend/interface.go b/backend/interface.go index 1fdacb4d7..f215c6733 100644 --- a/backend/interface.go +++ b/backend/interface.go @@ -43,6 +43,9 @@ type Backend interface { // Load returns the data stored in the backend for h at the given offset // and saves it in p. Load has the same semantics as io.ReaderAt. Load(h Handle, p []byte, off int64) (int, error) + + // Stat returns information about the blob identified by h. + Stat(h Handle) (BlobInfo, error) } // Lister implements listing data items stored in a backend. @@ -59,6 +62,11 @@ type Deleter interface { Delete() error } +// BlobInfo is returned by Stat() and contains information about a stored blob. +type BlobInfo struct { + Size int64 +} + // Blob is old. type Blob interface { io.Writer diff --git a/backend/local/local.go b/backend/local/local.go index 25ba26779..554710d37 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -252,6 +252,20 @@ func (b *Local) Load(h backend.Handle, p []byte, off int64) (n int, err error) { return io.ReadFull(f, p) } +// Stat returns information about a blob. +func (b *Local) Stat(h backend.Handle) (backend.BlobInfo, error) { + if err := h.Valid(); err != nil { + return backend.BlobInfo{}, err + } + + fi, err := os.Stat(filename(b.p, h.Type, h.Name)) + if err != nil { + return backend.BlobInfo{}, err + } + + return backend.BlobInfo{Size: fi.Size()}, nil +} + // Test returns true if a blob of the given type and name exists in the backend. func (b *Local) Test(t backend.Type, name string) (bool, error) { _, err := os.Stat(filename(b.p, t, name)) diff --git a/backend/mem/mem_backend.go b/backend/mem/mem_backend.go index e231c2ede..6e80735e3 100644 --- a/backend/mem/mem_backend.go +++ b/backend/mem/mem_backend.go @@ -49,6 +49,10 @@ func New() *MemoryBackend { return memLoad(be, h, p, off) } + be.MockBackend.StatFn = func(h backend.Handle) (backend.BlobInfo, error) { + return memStat(be, h) + } + be.MockBackend.RemoveFn = func(t backend.Type, name string) error { return memRemove(be, t, name) } @@ -195,6 +199,28 @@ func memLoad(be *MemoryBackend, h backend.Handle, p []byte, off int64) (int, err return n, nil } +func memStat(be *MemoryBackend, h backend.Handle) (backend.BlobInfo, error) { + be.m.Lock() + defer be.m.Unlock() + + if err := h.Valid(); err != nil { + return backend.BlobInfo{}, err + } + + if h.Type == backend.Config { + h.Name = "" + } + + debug.Log("MemoryBackend.Stat", "stat %v", h) + + e, ok := be.data[entry{h.Type, h.Name}] + if !ok { + return backend.BlobInfo{}, errors.New("no such data") + } + + return backend.BlobInfo{Size: int64(len(e))}, nil +} + func memRemove(be *MemoryBackend, t backend.Type, name string) error { be.m.Lock() defer be.m.Unlock() diff --git a/backend/mock_backend.go b/backend/mock_backend.go index 744cb7902..4d142f846 100644 --- a/backend/mock_backend.go +++ b/backend/mock_backend.go @@ -11,6 +11,7 @@ type MockBackend struct { CloseFn func() error CreateFn func() (Blob, error) LoadFn func(h Handle, p []byte, off int64) (int, error) + StatFn func(h Handle) (BlobInfo, error) GetReaderFn func(Type, string, uint, uint) (io.ReadCloser, error) ListFn func(Type, <-chan struct{}) <-chan string RemoveFn func(Type, string) error @@ -51,6 +52,14 @@ func (m *MockBackend) Load(h Handle, p []byte, off int64) (int, error) { return m.LoadFn(h, p, off) } +func (m *MockBackend) Stat(h Handle) (BlobInfo, error) { + if m.StatFn == nil { + return BlobInfo{}, errors.New("not implemented") + } + + return m.StatFn(h) +} + func (m *MockBackend) GetReader(t Type, name string, offset, len uint) (io.ReadCloser, error) { if m.GetReaderFn == nil { return nil, errors.New("not implemented") diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 2d3f89606..444fe2a73 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -191,6 +191,25 @@ func (be S3Backend) Load(h backend.Handle, p []byte, off int64) (int, error) { return io.ReadFull(obj, p) } +// Stat returns information about a blob. +func (be S3Backend) Stat(h backend.Handle) (backend.BlobInfo, error) { + debug.Log("s3.Stat", "%v") + path := s3path(h.Type, h.Name) + obj, err := be.client.GetObject(be.bucketname, path) + if err != nil { + debug.Log("s3.Stat", "GetObject() err %v", err) + return backend.BlobInfo{}, err + } + + fi, err := obj.Stat() + if err != nil { + debug.Log("s3.Stat", "Stat() err %v", err) + return backend.BlobInfo{}, err + } + + return backend.BlobInfo{Size: fi.Size}, nil +} + // Test returns true if a blob of the given type and name exists in the backend. func (be *S3Backend) Test(t backend.Type, name string) (bool, error) { found := false diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index a3d32e3d6..f23ec1db5 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -395,6 +395,20 @@ func (r *SFTP) Load(h backend.Handle, p []byte, off int64) (n int, err error) { return io.ReadFull(f, p) } +// Stat returns information about a blob. +func (r *SFTP) Stat(h backend.Handle) (backend.BlobInfo, error) { + if err := h.Valid(); err != nil { + return backend.BlobInfo{}, err + } + + fi, err := r.c.Lstat(r.filename(h.Type, h.Name)) + if err != nil { + return backend.BlobInfo{}, err + } + + return backend.BlobInfo{Size: fi.Size()}, nil +} + // Test returns true if a blob of the given type and name exists in the backend. func (r *SFTP) Test(t backend.Type, name string) (bool, error) { _, err := r.c.Lstat(r.filename(t, name)) diff --git a/backend/test/tests.go b/backend/test/tests.go index 2631fe168..bd12fdc4a 100644 --- a/backend/test/tests.go +++ b/backend/test/tests.go @@ -374,6 +374,13 @@ func TestWrite(t testing.TB) { t.Fatalf("data not equal") } + fi, err := b.Stat(backend.Handle{Type: backend.Data, Name: name}) + OK(t, err) + + if fi.Size != int64(len(data)) { + t.Fatalf("Stat() returned different size, want %q, got %d", len(data), fi.Size) + } + err = b.Remove(backend.Data, name) if err != nil { t.Fatalf("error removing item: %v", err) From 9209dcfa26e8f7ffea454a77b09f443eafd6b839 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 23:41:55 +0100 Subject: [PATCH 20/56] Add LoadAll() --- backend/utils.go | 18 ++++++++++++++++++ backend/utils_test.go | 44 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 backend/utils.go create mode 100644 backend/utils_test.go diff --git a/backend/utils.go b/backend/utils.go new file mode 100644 index 000000000..c40d35e12 --- /dev/null +++ b/backend/utils.go @@ -0,0 +1,18 @@ +package backend + +// LoadAll reads all data stored in the backend for the handle. The buffer buf +// is resized to accomodate all data in the blob. +func LoadAll(be Backend, h Handle, buf []byte) ([]byte, error) { + fi, err := be.Stat(h) + if err != nil { + return nil, err + } + + if fi.Size > int64(len(buf)) { + buf = make([]byte, int(fi.Size)) + } + + n, err := be.Load(h, buf, 0) + buf = buf[:n] + return buf, err +} diff --git a/backend/utils_test.go b/backend/utils_test.go new file mode 100644 index 000000000..426f866de --- /dev/null +++ b/backend/utils_test.go @@ -0,0 +1,44 @@ +package backend_test + +import ( + "bytes" + "math/rand" + "testing" + + "github.com/restic/restic/backend" + "github.com/restic/restic/backend/mem" + . "github.com/restic/restic/test" +) + +const KiB = 1 << 10 +const MiB = 1 << 20 + +func TestLoadAll(t *testing.T) { + b := mem.New() + + for i := 0; i < 20; i++ { + data := Random(23+i, rand.Intn(MiB)+500*KiB) + + id := backend.Hash(data) + + blob, err := b.Create() + OK(t, err) + + _, err = blob.Write([]byte(data)) + OK(t, err) + OK(t, blob.Finalize(backend.Data, id.String())) + + buf, err := backend.LoadAll(b, backend.Handle{Type: backend.Data, Name: id.String()}, nil) + OK(t, err) + + if len(buf) != len(data) { + t.Errorf("length of returned buffer does not match, want %d, got %d", len(data), len(buf)) + continue + } + + if !bytes.Equal(buf, data) { + t.Errorf("wrong data returned") + continue + } + } +} From 9bfa633187cf0d3af72247594ce52543555e49f1 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 23 Jan 2016 23:48:19 +0100 Subject: [PATCH 21/56] repository/key: Use Load() instead of GetReader() --- repository/key.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/repository/key.go b/repository/key.go index ea8fbd4c7..78665513b 100644 --- a/repository/key.go +++ b/repository/key.go @@ -119,17 +119,14 @@ func SearchKey(s *Repository, password string) (*Key, error) { // LoadKey loads a key from the backend. func LoadKey(s *Repository, name string) (k *Key, err error) { - // extract data from repo - rd, err := s.be.GetReader(backend.Key, name, 0, 0) + h := backend.Handle{Type: backend.Key, Name: name} + data, err := backend.LoadAll(s.be, h, nil) if err != nil { return nil, err } - defer closeOrErr(rd, &err) - // restore json - dec := json.NewDecoder(rd) - k = new(Key) - err = dec.Decode(k) + k = &Key{} + err = json.Unmarshal(data, k) if err != nil { return nil, err } From 3191778d3362ce4d18720747d290d8edd28eafab Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 00:12:09 +0100 Subject: [PATCH 22/56] repository: Use Load() instead of GetReader() --- repository/repository.go | 84 ++++++++-------------------------------- 1 file changed, 17 insertions(+), 67 deletions(-) diff --git a/repository/repository.go b/repository/repository.go index a0d7f71ef..f095898c3 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -56,24 +56,14 @@ func (r *Repository) PrefixLength(t backend.Type) (int, error) { func (r *Repository) LoadAndDecrypt(t backend.Type, id backend.ID) ([]byte, error) { debug.Log("Repo.Load", "load %v with id %v", t, id.Str()) - rd, err := r.be.GetReader(t, id.String(), 0, 0) + h := backend.Handle{Type: t, Name: id.String()} + buf, err := backend.LoadAll(r.be, h, nil) if err != nil { debug.Log("Repo.Load", "error loading %v: %v", id.Str(), err) return nil, err } - buf, err := ioutil.ReadAll(rd) - if err != nil { - return nil, err - } - - err = rd.Close() - if err != nil { - return nil, err - } - - // check hash - if !backend.Hash(buf).Equal(id) { + if t != backend.Config && !backend.Hash(buf).Equal(id) { return nil, errors.New("invalid data returned") } @@ -100,7 +90,9 @@ func (r *Repository) LoadBlob(t pack.BlobType, id backend.ID, plaintextBuf []byt plaintextBufSize := uint(cap(plaintextBuf)) if blob.PlaintextLength() > plaintextBufSize { - return nil, fmt.Errorf("buf is too small, need %d more bytes", blob.PlaintextLength()-plaintextBufSize) + debug.Log("Repo.LoadBlob", "need to expand buffer: want %d bytes, got %d", + blob.PlaintextLength(), plaintextBufSize) + plaintextBuf = make([]byte, blob.PlaintextLength()) } if blob.Type != t { @@ -111,22 +103,18 @@ func (r *Repository) LoadBlob(t pack.BlobType, id backend.ID, plaintextBuf []byt debug.Log("Repo.LoadBlob", "id %v found: %v", id.Str(), blob) // load blob from pack - rd, err := r.be.GetReader(backend.Data, blob.PackID.String(), blob.Offset, blob.Length) + h := backend.Handle{Type: backend.Data, Name: blob.PackID.String()} + ciphertextBuf := make([]byte, blob.Length) + n, err := r.be.Load(h, ciphertextBuf, int64(blob.Offset)) if err != nil { debug.Log("Repo.LoadBlob", "error loading blob %v: %v", blob, err) return nil, err } - // make buffer that is large enough for the complete blob - ciphertextBuf := make([]byte, blob.Length) - _, err = io.ReadFull(rd, ciphertextBuf) - if err != nil { - return nil, err - } - - err = rd.Close() - if err != nil { - return nil, err + if uint(n) != blob.Length { + debug.Log("Repo.LoadBlob", "error loading blob %v: wrong length returned, want %d, got %d", + blob.Length, uint(n)) + return nil, errors.New("wrong length returned") } // decrypt @@ -156,61 +144,23 @@ func closeOrErr(cl io.Closer, err *error) { // LoadJSONUnpacked decrypts the data and afterwards calls json.Unmarshal on // the item. func (r *Repository) LoadJSONUnpacked(t backend.Type, id backend.ID, item interface{}) (err error) { - // load blob from backend - rd, err := r.be.GetReader(t, id.String(), 0, 0) - if err != nil { - return err - } - defer closeOrErr(rd, &err) - - // decrypt - decryptRd, err := crypto.DecryptFrom(r.key, rd) - defer closeOrErr(decryptRd, &err) + buf, err := r.LoadAndDecrypt(t, id) if err != nil { return err } - // decode - decoder := json.NewDecoder(decryptRd) - err = decoder.Decode(item) - if err != nil { - return err - } - - return nil + return json.Unmarshal(buf, item) } // LoadJSONPack calls LoadBlob() to load a blob from the backend, decrypt the // data and afterwards call json.Unmarshal on the item. func (r *Repository) LoadJSONPack(t pack.BlobType, id backend.ID, item interface{}) (err error) { - // lookup pack - blob, err := r.idx.Lookup(id) + buf, err := r.LoadBlob(t, id, nil) if err != nil { return err } - // load blob from pack - rd, err := r.be.GetReader(backend.Data, blob.PackID.String(), blob.Offset, blob.Length) - if err != nil { - return err - } - defer closeOrErr(rd, &err) - - // decrypt - decryptRd, err := crypto.DecryptFrom(r.key, rd) - defer closeOrErr(decryptRd, &err) - if err != nil { - return err - } - - // decode - decoder := json.NewDecoder(decryptRd) - err = decoder.Decode(item) - if err != nil { - return err - } - - return nil + return json.Unmarshal(buf, item) } // LookupBlobSize returns the size of blob id. From 782a1bf7b05cb87d5f76c824ae1cd56320d19d45 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 00:12:17 +0100 Subject: [PATCH 23/56] repository: remove GetDecryptReader() --- repository/index.go | 6 +++--- repository/repository.go | 11 ----------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/repository/index.go b/repository/index.go index 7bb4a273e..093667336 100644 --- a/repository/index.go +++ b/repository/index.go @@ -1,6 +1,7 @@ package repository import ( + "bytes" "encoding/json" "errors" "fmt" @@ -564,13 +565,12 @@ func LoadIndexWithDecoder(repo *Repository, id string, fn func(io.Reader) (*Inde return nil, err } - rd, err := repo.GetDecryptReader(backend.Index, idxID.String()) + buf, err := repo.LoadAndDecrypt(backend.Index, idxID) if err != nil { return nil, err } - defer closeOrErr(rd, &err) - idx, err = fn(rd) + idx, err = fn(bytes.NewReader(buf)) if err != nil { debug.Log("LoadIndexWithDecoder", "error while decoding index %v: %v", id, err) return nil, err diff --git a/repository/repository.go b/repository/repository.go index f095898c3..a3ed9c60f 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -495,17 +495,6 @@ func LoadIndex(repo *Repository, id string) (*Index, error) { return nil, err } -// GetDecryptReader opens the file id stored in the backend and returns a -// reader that yields the decrypted content. The reader must be closed. -func (r *Repository) GetDecryptReader(t backend.Type, id string) (io.ReadCloser, error) { - rd, err := r.be.GetReader(t, id, 0, 0) - if err != nil { - return nil, err - } - - return newDecryptReadCloser(r.key, rd) -} - // SearchKey finds a key with the supplied password, afterwards the config is // read and parsed. func (r *Repository) SearchKey(password string) error { From 280d580ae2dce06a00c71cf579995a5551b0195f Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 00:14:15 +0100 Subject: [PATCH 24/56] checker: Use Load() instead of GetReader() --- checker/checker.go | 14 ++------------ checker/checker_test.go | 38 ++++++++++++-------------------------- 2 files changed, 14 insertions(+), 38 deletions(-) diff --git a/checker/checker.go b/checker/checker.go index 97577285d..97fe8df58 100644 --- a/checker/checker.go +++ b/checker/checker.go @@ -4,7 +4,6 @@ import ( "bytes" "errors" "fmt" - "io/ioutil" "sync" "github.com/restic/restic" @@ -647,17 +646,8 @@ func (c *Checker) CountPacks() uint64 { // checkPack reads a pack and checks the integrity of all blobs. func checkPack(r *repository.Repository, id backend.ID) error { debug.Log("Checker.checkPack", "checking pack %v", id.Str()) - rd, err := r.Backend().GetReader(backend.Data, id.String(), 0, 0) - if err != nil { - return err - } - - buf, err := ioutil.ReadAll(rd) - if err != nil { - return err - } - - err = rd.Close() + h := backend.Handle{Type: backend.Data, Name: id.String()} + buf, err := backend.LoadAll(r.Backend(), h, nil) if err != nil { return err } diff --git a/checker/checker_test.go b/checker/checker_test.go index 6118b987d..10b37e219 100644 --- a/checker/checker_test.go +++ b/checker/checker_test.go @@ -1,7 +1,7 @@ package checker_test import ( - "io" + "fmt" "math/rand" "path/filepath" "sort" @@ -213,24 +213,22 @@ func TestDuplicatePacksInIndex(t *testing.T) { // errorBackend randomly modifies data after reading. type errorBackend struct { backend.Backend + ProduceErrors bool } -func (b errorBackend) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { - rd, err := b.Backend.GetReader(t, name, offset, length) - if err != nil { - return rd, err - } +func (b errorBackend) Load(h backend.Handle, p []byte, off int64) (int, error) { + fmt.Printf("load %v\n", h) + n, err := b.Backend.Load(h, p, off) - if t != backend.Data { - return rd, err + if b.ProduceErrors { + induceError(p) } - - return backend.ReadCloser(faultReader{rd}), nil + return n, err } // induceError flips a bit in the slice. func induceError(data []byte) { - if rand.Float32() < 0.8 { + if rand.Float32() < 0.2 { return } @@ -238,20 +236,6 @@ func induceError(data []byte) { data[pos] ^= 1 } -// faultReader wraps a reader and randomly modifies data on read. -type faultReader struct { - rd io.Reader -} - -func (f faultReader) Read(p []byte) (int, error) { - n, err := f.rd.Read(p) - if n > 0 { - induceError(p) - } - - return n, err -} - func TestCheckerModifiedData(t *testing.T) { be := mem.New() @@ -263,7 +247,8 @@ func TestCheckerModifiedData(t *testing.T) { OK(t, err) t.Logf("archived as %v", id.Str()) - checkRepo := repository.New(errorBackend{be}) + beError := &errorBackend{Backend: be} + checkRepo := repository.New(beError) OK(t, checkRepo.SearchKey(TestPassword)) chkr := checker.New(checkRepo) @@ -277,6 +262,7 @@ func TestCheckerModifiedData(t *testing.T) { t.Errorf("expected no hints, got %v: %v", len(hints), hints) } + beError.ProduceErrors = true errFound := false for _, err := range checkPacks(chkr) { t.Logf("pack error: %v", err) From 61551b05913330147fda6456b2a2c37c278082d4 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 00:18:12 +0100 Subject: [PATCH 25/56] cmd_cat: Remove calls to GetReader() --- cmd/restic/cmd_cat.go | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index 2106fa306..1a7423d18 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "os" "github.com/restic/restic" @@ -101,20 +100,19 @@ func (cmd CmdCat) Execute(args []string) error { return nil case "key": - rd, err := repo.Backend().GetReader(backend.Key, id.String(), 0, 0) + h := backend.Handle{Type: backend.Key, Name: id.String()} + buf, err := backend.LoadAll(repo.Backend(), h, nil) if err != nil { return err } - dec := json.NewDecoder(rd) - - var key repository.Key - err = dec.Decode(&key) + key := &repository.Key{} + err = json.Unmarshal(buf, key) if err != nil { return err } - buf, err := json.MarshalIndent(&key, "", " ") + buf, err = json.MarshalIndent(&key, "", " ") if err != nil { return err } @@ -153,12 +151,13 @@ func (cmd CmdCat) Execute(args []string) error { switch tpe { case "pack": - rd, err := repo.Backend().GetReader(backend.Data, id.String(), 0, 0) + h := backend.Handle{Type: backend.Data, Name: id.String()} + buf, err := backend.LoadAll(repo.Backend(), h, nil) if err != nil { return err } - _, err = io.Copy(os.Stdout, rd) + _, err = os.Stdout.Write(buf) return err case "blob": From 2c3a6a6fa91c244d7f05918482c85a41254fa589 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 00:21:03 +0100 Subject: [PATCH 26/56] cmd_rebuild_index: Remove calls to GetReader() --- cmd/restic/cmd_rebuild_index.go | 29 +++++------------------------ 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/cmd/restic/cmd_rebuild_index.go b/cmd/restic/cmd_rebuild_index.go index 7a550354b..ee73992a6 100644 --- a/cmd/restic/cmd_rebuild_index.go +++ b/cmd/restic/cmd_rebuild_index.go @@ -2,8 +2,6 @@ package main import ( "bytes" - "io" - "io/ioutil" "github.com/restic/restic/backend" "github.com/restic/restic/debug" @@ -126,6 +124,7 @@ func (cmd CmdRebuildIndex) RebuildIndex() error { cmd.global.Printf("checking for additional packs\n") newPacks := 0 + var buf []byte for packID := range cmd.repo.List(backend.Data, done) { if packsDone.Has(packID) { continue @@ -134,27 +133,12 @@ func (cmd CmdRebuildIndex) RebuildIndex() error { debug.Log("RebuildIndex.RebuildIndex", "pack %v not indexed", packID.Str()) newPacks++ - rd, err := cmd.repo.Backend().GetReader(backend.Data, packID.String(), 0, 0) - if err != nil { - debug.Log("RebuildIndex.RebuildIndex", "GetReader returned error: %v", err) - return err - } + var err error - var readSeeker io.ReadSeeker - if r, ok := rd.(io.ReadSeeker); ok { - debug.Log("RebuildIndex.RebuildIndex", "reader is seekable") - readSeeker = r - } else { - debug.Log("RebuildIndex.RebuildIndex", "reader is not seekable, loading contents to ram") - buf, err := ioutil.ReadAll(rd) - if err != nil { - return err - } + h := backend.Handle{Type: backend.Data, Name: packID.String()} + buf, err = backend.LoadAll(cmd.repo.Backend(), h, buf) - readSeeker = bytes.NewReader(buf) - } - - up, err := pack.NewUnpacker(cmd.repo.Key(), readSeeker) + up, err := pack.NewUnpacker(cmd.repo.Key(), bytes.NewReader(buf)) if err != nil { debug.Log("RebuildIndex.RebuildIndex", "error while unpacking pack %v", packID.Str()) return err @@ -171,9 +155,6 @@ func (cmd CmdRebuildIndex) RebuildIndex() error { }) } - err = rd.Close() - debug.Log("RebuildIndex.RebuildIndex", "error closing reader for pack %v: %v", packID.Str(), err) - if repository.IndexFull(combinedIndex) { combinedIndex, err = cmd.storeIndex(combinedIndex) if err != nil { From adbe9e2e1ceb29eef2880fa06343f6fae39eb6b9 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 01:00:27 +0100 Subject: [PATCH 27/56] backend: Remove GetReader --- backend/generic.go | 37 -------------- backend/interface.go | 4 -- backend/local/backend_test.go | 7 --- backend/local/local.go | 27 ---------- backend/mem/backend_test.go | 7 --- backend/mem/mem_backend.go | 36 ------------- backend/mock_backend.go | 32 ++++-------- backend/s3/backend_test.go | 7 --- backend/s3/s3.go | 25 --------- backend/sftp/backend_test.go | 7 --- backend/sftp/sftp.go | 22 -------- backend/test/backend_test.go | 7 --- backend/test/tests.go | 96 ++++++----------------------------- 13 files changed, 26 insertions(+), 288 deletions(-) diff --git a/backend/generic.go b/backend/generic.go index 4c736a6b2..d8c72b9e0 100644 --- a/backend/generic.go +++ b/backend/generic.go @@ -3,7 +3,6 @@ package backend import ( "crypto/sha256" "errors" - "io" ) const ( @@ -82,39 +81,3 @@ outer: return IDSize, nil } - -// wrap around io.LimitedReader that implements io.ReadCloser -type blobReader struct { - cl io.Closer - rd io.Reader - closed bool -} - -func (l *blobReader) Read(p []byte) (int, error) { - n, err := l.rd.Read(p) - if err == io.EOF { - l.Close() - } - - return n, err -} - -func (l *blobReader) Close() error { - if l == nil { - return nil - } - - if !l.closed { - err := l.cl.Close() - l.closed = true - return err - } - - return nil -} - -// LimitReadCloser returns a new reader wraps r in an io.LimitReader, but also -// implements the Close() method. -func LimitReadCloser(r io.ReadCloser, n int64) *blobReader { - return &blobReader{cl: r, rd: io.LimitReader(r, n)} -} diff --git a/backend/interface.go b/backend/interface.go index f215c6733..24424dd96 100644 --- a/backend/interface.go +++ b/backend/interface.go @@ -25,10 +25,6 @@ type Backend interface { // has been called on the returned Blob. Create() (Blob, error) - // GetReader returns an io.ReadCloser for the Blob with the given name of - // type t at offset and length. - GetReader(t Type, name string, offset, length uint) (io.ReadCloser, error) - // Test a boolean value whether a Blob with the name and type exists. Test(t Type, name string) (bool, error) diff --git a/backend/local/backend_test.go b/backend/local/backend_test.go index 8cc244e85..c6032be6b 100644 --- a/backend/local/backend_test.go +++ b/backend/local/backend_test.go @@ -44,13 +44,6 @@ func TestLocalBackendConfig(t *testing.T) { test.TestConfig(t) } -func TestLocalBackendGetReader(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestGetReader(t) -} - func TestLocalBackendLoad(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/local/local.go b/backend/local/local.go index 554710d37..0d12e4109 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -196,33 +196,6 @@ func dirname(base string, t backend.Type, name string) string { return filepath.Join(base, n) } -// GetReader returns an io.ReadCloser for the Blob with the given name of -// type t at offset and length. If length is 0, the reader reads until EOF. -func (b *Local) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { - f, err := os.Open(filename(b.p, t, name)) - if err != nil { - return nil, err - } - - b.mu.Lock() - open, _ := b.open[filename(b.p, t, name)] - b.open[filename(b.p, t, name)] = append(open, f) - b.mu.Unlock() - - if offset > 0 { - _, err = f.Seek(int64(offset), 0) - if err != nil { - return nil, err - } - } - - if length == 0 { - return f, nil - } - - return backend.LimitReadCloser(f, int64(length)), nil -} - // Load returns the data stored in the backend for h at the given offset // and saves it in p. Load has the same semantics as io.ReaderAt. func (b *Local) Load(h backend.Handle, p []byte, off int64) (n int, err error) { diff --git a/backend/mem/backend_test.go b/backend/mem/backend_test.go index 233b30f60..681d0a680 100644 --- a/backend/mem/backend_test.go +++ b/backend/mem/backend_test.go @@ -44,13 +44,6 @@ func TestMemBackendConfig(t *testing.T) { test.TestConfig(t) } -func TestMemBackendGetReader(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestGetReader(t) -} - func TestMemBackendLoad(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/mem/mem_backend.go b/backend/mem/mem_backend.go index 6e80735e3..b999efd1d 100644 --- a/backend/mem/mem_backend.go +++ b/backend/mem/mem_backend.go @@ -41,10 +41,6 @@ func New() *MemoryBackend { return memCreate(be) } - be.MockBackend.GetReaderFn = func(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { - return memGetReader(be, t, name, offset, length) - } - be.MockBackend.LoadFn = func(h backend.Handle, p []byte, off int64) (int, error) { return memLoad(be, h, p, off) } @@ -133,38 +129,6 @@ func memCreate(be *MemoryBackend) (backend.Blob, error) { return blob, nil } -func memGetReader(be *MemoryBackend, t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { - be.m.Lock() - defer be.m.Unlock() - - if t == backend.Config { - name = "" - } - - debug.Log("MemoryBackend.GetReader", "get %v %v offset %v len %v", t, name, offset, length) - - if _, ok := be.data[entry{t, name}]; !ok { - return nil, errors.New("no such data") - } - - buf := be.data[entry{t, name}] - if offset > uint(len(buf)) { - return nil, errors.New("offset beyond end of file") - } - - buf = buf[offset:] - - if length > 0 { - if length > uint(len(buf)) { - length = uint(len(buf)) - } - - buf = buf[:length] - } - - return backend.ReadCloser(bytes.NewReader(buf)), nil -} - func memLoad(be *MemoryBackend, h backend.Handle, p []byte, off int64) (int, error) { be.m.Lock() defer be.m.Unlock() diff --git a/backend/mock_backend.go b/backend/mock_backend.go index 4d142f846..1075292d6 100644 --- a/backend/mock_backend.go +++ b/backend/mock_backend.go @@ -1,23 +1,19 @@ package backend -import ( - "errors" - "io" -) +import "errors" // MockBackend implements a backend whose functions can be specified. This // should only be used for tests. type MockBackend struct { - CloseFn func() error - CreateFn func() (Blob, error) - LoadFn func(h Handle, p []byte, off int64) (int, error) - StatFn func(h Handle) (BlobInfo, error) - GetReaderFn func(Type, string, uint, uint) (io.ReadCloser, error) - ListFn func(Type, <-chan struct{}) <-chan string - RemoveFn func(Type, string) error - TestFn func(Type, string) (bool, error) - DeleteFn func() error - LocationFn func() string + CloseFn func() error + CreateFn func() (Blob, error) + LoadFn func(h Handle, p []byte, off int64) (int, error) + StatFn func(h Handle) (BlobInfo, error) + ListFn func(Type, <-chan struct{}) <-chan string + RemoveFn func(Type, string) error + TestFn func(Type, string) (bool, error) + DeleteFn func() error + LocationFn func() string } func (m *MockBackend) Close() error { @@ -60,14 +56,6 @@ func (m *MockBackend) Stat(h Handle) (BlobInfo, error) { return m.StatFn(h) } -func (m *MockBackend) GetReader(t Type, name string, offset, len uint) (io.ReadCloser, error) { - if m.GetReaderFn == nil { - return nil, errors.New("not implemented") - } - - return m.GetReaderFn(t, name, offset, len) -} - func (m *MockBackend) List(t Type, done <-chan struct{}) <-chan string { if m.ListFn == nil { ch := make(chan string) diff --git a/backend/s3/backend_test.go b/backend/s3/backend_test.go index d0d15f51a..cc981cf18 100644 --- a/backend/s3/backend_test.go +++ b/backend/s3/backend_test.go @@ -44,13 +44,6 @@ func TestS3BackendConfig(t *testing.T) { test.TestConfig(t) } -func TestS3BackendGetReader(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestGetReader(t) -} - func TestS3BackendLoad(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 444fe2a73..a46cdb22d 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -145,31 +145,6 @@ func (be *S3Backend) Create() (backend.Blob, error) { return &blob, nil } -// GetReader returns an io.ReadCloser for the Blob with the given name of -// type t at offset and length. If length is 0, the reader reads until EOF. -func (be *S3Backend) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { - debug.Log("s3.GetReader", "%v %v, offset %v len %v", t, name, offset, length) - path := s3path(t, name) - obj, err := be.client.GetObject(be.bucketname, path) - if err != nil { - debug.Log("s3.GetReader", " err %v", err) - return nil, err - } - - if offset > 0 { - _, err = obj.Seek(int64(offset), 0) - if err != nil { - return nil, err - } - } - - if length == 0 { - return obj, nil - } - - return backend.LimitReadCloser(obj, int64(length)), nil -} - // Load returns the data stored in the backend for h at the given offset // and saves it in p. Load has the same semantics as io.ReaderAt. func (be S3Backend) Load(h backend.Handle, p []byte, off int64) (int, error) { diff --git a/backend/sftp/backend_test.go b/backend/sftp/backend_test.go index ef4ac9129..217f34b11 100644 --- a/backend/sftp/backend_test.go +++ b/backend/sftp/backend_test.go @@ -44,13 +44,6 @@ func TestSftpBackendConfig(t *testing.T) { test.TestConfig(t) } -func TestSftpBackendGetReader(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestGetReader(t) -} - func TestSftpBackendLoad(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index f23ec1db5..e40a3cd00 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -344,28 +344,6 @@ func (r *SFTP) dirname(t backend.Type, name string) string { return Join(r.p, n) } -// GetReader returns an io.ReadCloser for the Blob with the given name of -// type t at offset and length. If length is 0, the reader reads until EOF. -func (r *SFTP) GetReader(t backend.Type, name string, offset, length uint) (io.ReadCloser, error) { - f, err := r.c.Open(r.filename(t, name)) - if err != nil { - return nil, err - } - - if offset > 0 { - _, err = f.Seek(int64(offset), 0) - if err != nil { - return nil, err - } - } - - if length == 0 { - return f, nil - } - - return backend.LimitReadCloser(f, int64(length)), nil -} - // Load returns the data stored in the backend for h at the given offset // and saves it in p. Load has the same semantics as io.ReaderAt. func (r *SFTP) Load(h backend.Handle, p []byte, off int64) (n int, err error) { diff --git a/backend/test/backend_test.go b/backend/test/backend_test.go index 44b330a0a..733fecaaa 100644 --- a/backend/test/backend_test.go +++ b/backend/test/backend_test.go @@ -44,13 +44,6 @@ func TestTestBackendConfig(t *testing.T) { test.TestConfig(t) } -func TestTestBackendGetReader(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestGetReader(t) -} - func TestTestBackendLoad(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/test/tests.go b/backend/test/tests.go index bd12fdc4a..009ef1172 100644 --- a/backend/test/tests.go +++ b/backend/test/tests.go @@ -152,7 +152,7 @@ func TestConfig(t testing.TB) { var testString = "Config" // create config and read it back - _, err := b.GetReader(backend.Config, "", 0, 0) + _, err := backend.LoadAll(b, backend.Handle{Type: backend.Config}, nil) if err == nil { t.Fatalf("did not get expected error for non-existing config") } @@ -175,76 +175,17 @@ func TestConfig(t testing.TB) { // try accessing the config with different names, should all return the // same config for _, name := range []string{"", "foo", "bar", "0000000000000000000000000000000000000000000000000000000000000000"} { - rd, err := b.GetReader(backend.Config, name, 0, 0) + buf, err := backend.LoadAll(b, backend.Handle{Type: backend.Config}, nil) if err != nil { t.Fatalf("unable to read config with name %q: %v", name, err) } - buf, err := ioutil.ReadAll(rd) - if err != nil { - t.Fatalf("read config error: %v", err) - } - - err = rd.Close() - if err != nil { - t.Fatalf("close error: %v", err) - } - if string(buf) != testString { t.Fatalf("wrong data returned, want %q, got %q", testString, string(buf)) } } } -// TestGetReader tests various ways the GetReader function can be called. -func TestGetReader(t testing.TB) { - b := open(t) - defer close(t) - - length := rand.Intn(1<<24) + 2000 - - data := make([]byte, length) - _, err := io.ReadFull(crand.Reader, data) - OK(t, err) - - blob, err := b.Create() - OK(t, err) - - id := backend.Hash(data) - - _, err = blob.Write([]byte(data)) - OK(t, err) - OK(t, blob.Finalize(backend.Data, id.String())) - - for i := 0; i < 500; i++ { - l := rand.Intn(length + 2000) - o := rand.Intn(length + 2000) - - d := data - if o < len(d) { - d = d[o:] - } else { - o = len(d) - d = d[:0] - } - - if l > 0 && l < len(d) { - d = d[:l] - } - - rd, err := b.GetReader(backend.Data, id.String(), uint(o), uint(l)) - OK(t, err) - buf, err := ioutil.ReadAll(rd) - OK(t, err) - - if !bytes.Equal(buf, d) { - t.Fatalf("data not equal") - } - } - - OK(t, b.Remove(backend.Data, id.String())) -} - // TestLoad tests the backend's Load function. func TestLoad(t testing.TB) { b := open(t) @@ -360,12 +301,8 @@ func TestWrite(t testing.TB) { name := fmt.Sprintf("%s-%d", id, i) OK(t, blob.Finalize(backend.Data, name)) - rd, err := b.GetReader(backend.Data, name, 0, 0) + buf, err := backend.LoadAll(b, backend.Handle{Type: backend.Data, Name: name}, nil) OK(t, err) - - buf, err := ioutil.ReadAll(rd) - OK(t, err) - if len(buf) != len(data) { t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf)) } @@ -436,12 +373,13 @@ func TestBackend(t testing.TB) { OK(t, err) Assert(t, !ret, "blob was found to exist before creating") - // try to open not existing blob - _, err = b.GetReader(tpe, id.String(), 0, 0) + // try to stat a not existing blob + h := backend.Handle{Type: tpe, Name: id.String()} + _, err = b.Stat(h) Assert(t, err != nil, "blob data could be extracted before creation") // try to read not existing blob - _, err = b.GetReader(tpe, id.String(), 0, 1) + _, err = b.Load(h, nil, 0) Assert(t, err != nil, "blob reader could be obtained before creation") // try to get string out, should fail @@ -454,24 +392,22 @@ func TestBackend(t testing.TB) { for _, test := range testStrings { store(t, b, tpe, []byte(test.data)) - // test GetReader() - rd, err := b.GetReader(tpe, test.id, 0, uint(len(test.data))) + // test Load() + h := backend.Handle{Type: tpe, Name: test.id} + buf, err := backend.LoadAll(b, h, nil) OK(t, err) - Assert(t, rd != nil, "GetReader() returned nil") - - read(t, rd, []byte(test.data)) - OK(t, rd.Close()) + Equals(t, test.data, string(buf)) // try to read it out with an offset and a length start := 1 end := len(test.data) - 2 length := end - start - rd, err = b.GetReader(tpe, test.id, uint(start), uint(length)) - OK(t, err) - Assert(t, rd != nil, "GetReader() returned nil") - read(t, rd, []byte(test.data[start:end])) - OK(t, rd.Close()) + buf2 := make([]byte, length) + n, err := b.Load(h, buf2, int64(start)) + OK(t, err) + Equals(t, length, n) + Equals(t, test.data[start:end], string(buf2)) } // test adding the first file again From ed172c06e072bd4c08d2103143694aeef5151a12 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 01:15:35 +0100 Subject: [PATCH 28/56] backends: Add Save() function --- backend/interface.go | 3 +++ backend/local/local.go | 27 +++++++++++++++++++++++++++ backend/mem/mem_backend.go | 26 +++++++++++++++++++++++--- backend/mock_backend.go | 9 +++++++++ backend/s3/s3.go | 27 +++++++++++++++++++++++++++ backend/sftp/sftp.go | 23 +++++++++++++++++++++++ 6 files changed, 112 insertions(+), 3 deletions(-) diff --git a/backend/interface.go b/backend/interface.go index 24424dd96..9fff4bf1e 100644 --- a/backend/interface.go +++ b/backend/interface.go @@ -40,6 +40,9 @@ type Backend interface { // and saves it in p. Load has the same semantics as io.ReaderAt. Load(h Handle, p []byte, off int64) (int, error) + // Save stores the data in the backend under the given handle. + Save(h Handle, p []byte) error + // Stat returns information about the blob identified by h. Stat(h Handle) (BlobInfo, error) } diff --git a/backend/local/local.go b/backend/local/local.go index 0d12e4109..4183ff474 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -225,6 +225,33 @@ func (b *Local) Load(h backend.Handle, p []byte, off int64) (n int, err error) { return io.ReadFull(f, p) } +// Save stores data in the backend at the handle. +func (b *Local) Save(h backend.Handle, p []byte) (err error) { + if err := h.Valid(); err != nil { + return err + } + + f, err := os.Create(filename(b.p, h.Type, h.Name)) + if err != nil { + return err + } + + n, err := f.Write(p) + if err != nil { + return err + } + + if n != len(p) { + return errors.New("not all bytes writen") + } + + if err = f.Sync(); err != nil { + return err + } + + return f.Close() +} + // Stat returns information about a blob. func (b *Local) Stat(h backend.Handle) (backend.BlobInfo, error) { if err := h.Valid(); err != nil { diff --git a/backend/mem/mem_backend.go b/backend/mem/mem_backend.go index b999efd1d..0de0c0837 100644 --- a/backend/mem/mem_backend.go +++ b/backend/mem/mem_backend.go @@ -130,13 +130,13 @@ func memCreate(be *MemoryBackend) (backend.Blob, error) { } func memLoad(be *MemoryBackend, h backend.Handle, p []byte, off int64) (int, error) { - be.m.Lock() - defer be.m.Unlock() - if err := h.Valid(); err != nil { return 0, err } + be.m.Lock() + defer be.m.Unlock() + if h.Type == backend.Config { h.Name = "" } @@ -163,6 +163,26 @@ func memLoad(be *MemoryBackend, h backend.Handle, p []byte, off int64) (int, err return n, nil } +func memSave(be *MemoryBackend, h backend.Handle, p []byte) error { + if err := h.Valid(); err != nil { + return err + } + + be.m.Lock() + defer be.m.Unlock() + + if h.Type == backend.Config { + h.Name = "" + } + + debug.Log("MemoryBackend.Save", "save %v bytes at %v", len(p), h) + buf := make([]byte, len(p)) + copy(buf, p) + be.data[entry{h.Type, h.Name}] = buf + + return nil +} + func memStat(be *MemoryBackend, h backend.Handle) (backend.BlobInfo, error) { be.m.Lock() defer be.m.Unlock() diff --git a/backend/mock_backend.go b/backend/mock_backend.go index 1075292d6..9e692883f 100644 --- a/backend/mock_backend.go +++ b/backend/mock_backend.go @@ -8,6 +8,7 @@ type MockBackend struct { CloseFn func() error CreateFn func() (Blob, error) LoadFn func(h Handle, p []byte, off int64) (int, error) + SaveFn func(h Handle, p []byte) error StatFn func(h Handle) (BlobInfo, error) ListFn func(Type, <-chan struct{}) <-chan string RemoveFn func(Type, string) error @@ -48,6 +49,14 @@ func (m *MockBackend) Load(h Handle, p []byte, off int64) (int, error) { return m.LoadFn(h, p, off) } +func (m *MockBackend) Save(h Handle, p []byte) error { + if m.SaveFn == nil { + return errors.New("not implemented") + } + + return m.SaveFn(h, p) +} + func (m *MockBackend) Stat(h Handle) (BlobInfo, error) { if m.StatFn == nil { return BlobInfo{}, errors.New("not implemented") diff --git a/backend/s3/s3.go b/backend/s3/s3.go index a46cdb22d..9fd5c6152 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -163,9 +163,36 @@ func (be S3Backend) Load(h backend.Handle, p []byte, off int64) (int, error) { } } + <-be.connChan + defer func() { + be.connChan <- struct{}{} + }() return io.ReadFull(obj, p) } +// Save stores data in the backend at the handle. +func (be S3Backend) Save(h backend.Handle, p []byte) (err error) { + if err := h.Valid(); err != nil { + return err + } + + debug.Log("s3.Save", "%v bytes at %d", len(p), h) + + path := s3path(h.Type, h.Name) + + <-be.connChan + defer func() { + be.connChan <- struct{}{} + }() + + debug.Log("s3.Save", "PutObject(%v, %v, %v, %v)", + be.bucketname, path, int64(len(p)), "binary/octet-stream") + n, err := be.client.PutObject(be.bucketname, path, bytes.NewReader(p), "binary/octet-stream") + debug.Log("s3.Save", "%v -> %v bytes, err %#v", path, n, err) + + return err +} + // Stat returns information about a blob. func (be S3Backend) Stat(h backend.Handle) (backend.BlobInfo, error) { debug.Log("s3.Stat", "%v") diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index e40a3cd00..c5017bdfc 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -373,6 +373,29 @@ func (r *SFTP) Load(h backend.Handle, p []byte, off int64) (n int, err error) { return io.ReadFull(f, p) } +// Save stores data in the backend at the handle. +func (r *SFTP) Save(h backend.Handle, p []byte) (err error) { + if err := h.Valid(); err != nil { + return err + } + + f, err := r.c.Create(r.filename(h.Type, h.Name)) + if err != nil { + return err + } + + n, err := f.Write(p) + if err != nil { + return err + } + + if n != len(p) { + return errors.New("not all bytes writen") + } + + return f.Close() +} + // Stat returns information about a blob. func (r *SFTP) Stat(h backend.Handle) (backend.BlobInfo, error) { if err := h.Valid(); err != nil { From 54f88606121d504cccf493ea789478570ea65e42 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 16:59:38 +0100 Subject: [PATCH 29/56] backends: Add Save() --- backend/handle.go | 2 +- backend/local/backend_test.go | 14 ++++++ backend/local/local.go | 58 +++++++++++++++++++++-- backend/mem/backend_test.go | 14 ++++++ backend/mem/mem_backend.go | 4 ++ backend/s3/backend_test.go | 14 ++++++ backend/sftp/backend_test.go | 14 ++++++ backend/sftp/sftp.go | 23 ++++++--- backend/test/backend_test.go | 14 ++++++ backend/test/tests.go | 87 +++++++++++++++++++++++++++++++++++ 10 files changed, 233 insertions(+), 11 deletions(-) diff --git a/backend/handle.go b/backend/handle.go index 9c7a443ff..6bdf6af23 100644 --- a/backend/handle.go +++ b/backend/handle.go @@ -33,7 +33,7 @@ func (h Handle) Valid() error { case Index: case Config: default: - return fmt.Errorf("invalid config %q", h.Type) + return fmt.Errorf("invalid Type %q", h.Type) } if h.Type == Config { diff --git a/backend/local/backend_test.go b/backend/local/backend_test.go index c6032be6b..5617ab74a 100644 --- a/backend/local/backend_test.go +++ b/backend/local/backend_test.go @@ -58,6 +58,20 @@ func TestLocalBackendWrite(t *testing.T) { test.TestWrite(t) } +func TestLocalBackendSave(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSave(t) +} + +func TestLocalBackendSaveFilenames(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSaveFilenames(t) +} + func TestLocalBackendBackend(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/local/local.go b/backend/local/local.go index 4183ff474..5f836a8a5 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -11,6 +11,7 @@ import ( "sync" "github.com/restic/restic/backend" + "github.com/restic/restic/debug" ) var ErrWrongData = errors.New("wrong data returned by backend, checksum does not match") @@ -231,12 +232,14 @@ func (b *Local) Save(h backend.Handle, p []byte) (err error) { return err } - f, err := os.Create(filename(b.p, h.Type, h.Name)) + tmpfile, err := ioutil.TempFile(filepath.Join(b.p, backend.Paths.Temp), "temp-") if err != nil { return err } - n, err := f.Write(p) + debug.Log("local.Save", "save %v (%d bytes) to %v", h, len(p), tmpfile.Name()) + + n, err := tmpfile.Write(p) if err != nil { return err } @@ -245,11 +248,58 @@ func (b *Local) Save(h backend.Handle, p []byte) (err error) { return errors.New("not all bytes writen") } - if err = f.Sync(); err != nil { + if err = tmpfile.Sync(); err != nil { return err } - return f.Close() + err = tmpfile.Close() + if err != nil { + return err + } + + f := filename(b.p, h.Type, h.Name) + + // create directories if necessary, ignore errors + if h.Type == backend.Data { + os.MkdirAll(filepath.Dir(f), backend.Modes.Dir) + } + + // test if new path already exists + if _, err := os.Stat(f); err == nil { + return fmt.Errorf("Rename(): file %v already exists", f) + } + + err = os.Rename(tmpfile.Name(), f) + debug.Log("local.Save", "save %v: rename %v -> %v: %v", + h, filepath.Base(tmpfile.Name()), filepath.Base(f), err) + + if err != nil { + return err + } + + // set mode to read-only + fi, err := os.Stat(f) + if err != nil { + return err + } + + err = setNewFileMode(f, fi) + if err != nil { + return err + } + + // try to flush directory + d, err := os.Open(filepath.Dir(f)) + if err != nil { + return err + } + + err = d.Sync() + if err != nil { + return err + } + + return d.Close() } // Stat returns information about a blob. diff --git a/backend/mem/backend_test.go b/backend/mem/backend_test.go index 681d0a680..9b1a602a1 100644 --- a/backend/mem/backend_test.go +++ b/backend/mem/backend_test.go @@ -58,6 +58,20 @@ func TestMemBackendWrite(t *testing.T) { test.TestWrite(t) } +func TestMemBackendSave(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSave(t) +} + +func TestMemBackendSaveFilenames(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSaveFilenames(t) +} + func TestMemBackendBackend(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/mem/mem_backend.go b/backend/mem/mem_backend.go index 0de0c0837..91831dc5e 100644 --- a/backend/mem/mem_backend.go +++ b/backend/mem/mem_backend.go @@ -45,6 +45,10 @@ func New() *MemoryBackend { return memLoad(be, h, p, off) } + be.MockBackend.SaveFn = func(h backend.Handle, p []byte) error { + return memSave(be, h, p) + } + be.MockBackend.StatFn = func(h backend.Handle) (backend.BlobInfo, error) { return memStat(be, h) } diff --git a/backend/s3/backend_test.go b/backend/s3/backend_test.go index cc981cf18..4bd4d6bee 100644 --- a/backend/s3/backend_test.go +++ b/backend/s3/backend_test.go @@ -58,6 +58,20 @@ func TestS3BackendWrite(t *testing.T) { test.TestWrite(t) } +func TestS3BackendSave(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSave(t) +} + +func TestS3BackendSaveFilenames(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSaveFilenames(t) +} + func TestS3BackendBackend(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/sftp/backend_test.go b/backend/sftp/backend_test.go index 217f34b11..ca268b3a3 100644 --- a/backend/sftp/backend_test.go +++ b/backend/sftp/backend_test.go @@ -58,6 +58,20 @@ func TestSftpBackendWrite(t *testing.T) { test.TestWrite(t) } +func TestSftpBackendSave(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSave(t) +} + +func TestSftpBackendSaveFilenames(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSaveFilenames(t) +} + func TestSftpBackendBackend(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index c5017bdfc..a7ad4955e 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -15,6 +15,7 @@ import ( "github.com/juju/errors" "github.com/pkg/sftp" "github.com/restic/restic/backend" + "github.com/restic/restic/debug" ) const ( @@ -379,12 +380,10 @@ func (r *SFTP) Save(h backend.Handle, p []byte) (err error) { return err } - f, err := r.c.Create(r.filename(h.Type, h.Name)) - if err != nil { - return err - } + filename, tmpfile, err := r.tempFile() + debug.Log("sftp.Save", "save %v (%d bytes) to %v", h, len(p), filename) - n, err := f.Write(p) + n, err := tmpfile.Write(p) if err != nil { return err } @@ -393,7 +392,19 @@ func (r *SFTP) Save(h backend.Handle, p []byte) (err error) { return errors.New("not all bytes writen") } - return f.Close() + err = tmpfile.Close() + if err != nil { + return err + } + + err = r.renameFile(filename, h.Type, h.Name) + debug.Log("sftp.Save", "save %v: rename %v: %v", + h, filepath.Base(filename), err) + if err != nil { + return fmt.Errorf("sftp: renameFile: %v", err) + } + + return nil } // Stat returns information about a blob. diff --git a/backend/test/backend_test.go b/backend/test/backend_test.go index 733fecaaa..e9c19680d 100644 --- a/backend/test/backend_test.go +++ b/backend/test/backend_test.go @@ -58,6 +58,20 @@ func TestTestBackendWrite(t *testing.T) { test.TestWrite(t) } +func TestTestBackendSave(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSave(t) +} + +func TestTestBackendSaveFilenames(t *testing.T) { + if SkipMessage != "" { + t.Skip(SkipMessage) + } + test.TestSaveFilenames(t) +} + func TestTestBackendBackend(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/test/tests.go b/backend/test/tests.go index 009ef1172..f91b59da1 100644 --- a/backend/test/tests.go +++ b/backend/test/tests.go @@ -325,6 +325,93 @@ func TestWrite(t testing.TB) { } } +// TestSave tests saving data in the backend. +func TestSave(t testing.TB) { + b := open(t) + defer close(t) + + length := rand.Intn(1<<23) + 2000 + data := make([]byte, length) + + for i := 0; i < 10; i++ { + _, err := io.ReadFull(crand.Reader, data) + OK(t, err) + id := backend.Hash(data) + + h := backend.Handle{ + Type: backend.Data, + Name: fmt.Sprintf("%s-%d", id, i), + } + err = b.Save(h, data) + OK(t, err) + + buf, err := backend.LoadAll(b, h, nil) + OK(t, err) + if len(buf) != len(data) { + t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf)) + } + + if !bytes.Equal(buf, data) { + t.Fatalf("data not equal") + } + + fi, err := b.Stat(h) + OK(t, err) + + if fi.Size != int64(len(data)) { + t.Fatalf("Stat() returned different size, want %q, got %d", len(data), fi.Size) + } + + err = b.Remove(h.Type, h.Name) + if err != nil { + t.Fatalf("error removing item: %v", err) + } + } +} + +var filenameTests = []struct { + name string + data string +}{ + {"1dfc6bc0f06cb255889e9ea7860a5753e8eb9665c9a96627971171b444e3113e", "x"}, + {"foobar", "foobar"}, + { + "1dfc6bc0f06cb255889e9ea7860a5753e8eb9665c9a96627971171b444e3113e4bf8f2d9144cc5420a80f04a4880ad6155fc58903a4fb6457c476c43541dcaa6-5", + "foobar content of data blob", + }, +} + +// TestSaveFilenames tests saving data with various file names in the backend. +func TestSaveFilenames(t testing.TB) { + b := open(t) + defer close(t) + + for i, test := range filenameTests { + h := backend.Handle{Name: test.name, Type: backend.Data} + err := b.Save(h, []byte(test.data)) + if err != nil { + t.Errorf("test %d failed: Save() returned %v", i, err) + continue + } + + buf, err := backend.LoadAll(b, h, nil) + if err != nil { + t.Errorf("test %d failed: Load() returned %v", i, err) + continue + } + + if !bytes.Equal(buf, []byte(test.data)) { + t.Errorf("test %d: returned wrong bytes", i) + } + + err = b.Remove(h.Type, h.Name) + if err != nil { + t.Errorf("test %d failed: Remove() returned %v", i, err) + continue + } + } +} + var testStrings = []struct { id string data string From 4735a7f9b57738dbddec96b02ffe093034311364 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 17:46:18 +0100 Subject: [PATCH 30/56] Improve random reader for tests --- backend/test/tests.go | 25 ++++++----------- crypto/crypto_test.go | 27 +++++++------------ test/helpers.go | 62 ++++++++++++++++++++++++++++++++++++------- 3 files changed, 69 insertions(+), 45 deletions(-) diff --git a/backend/test/tests.go b/backend/test/tests.go index f91b59da1..b3557d10d 100644 --- a/backend/test/tests.go +++ b/backend/test/tests.go @@ -2,7 +2,6 @@ package test import ( "bytes" - crand "crypto/rand" "fmt" "io" "io/ioutil" @@ -203,12 +202,7 @@ func TestLoad(t testing.TB) { length := rand.Intn(1<<24) + 2000 - data := make([]byte, length) - _, err = io.ReadFull(crand.Reader, data) - if err != nil { - t.Fatalf("reading random data failed: %v", err) - } - + data := Random(23, length) id := backend.Hash(data) blob, err := b.Create() @@ -273,9 +267,7 @@ func TestWrite(t testing.TB) { length := rand.Intn(1<<23) + 2000 - data := make([]byte, length) - _, err := io.ReadFull(crand.Reader, data) - OK(t, err) + data := Random(23, length) id := backend.Hash(data) for i := 0; i < 10; i++ { @@ -329,20 +321,19 @@ func TestWrite(t testing.TB) { func TestSave(t testing.TB) { b := open(t) defer close(t) - - length := rand.Intn(1<<23) + 2000 - data := make([]byte, length) + var id backend.ID for i := 0; i < 10; i++ { - _, err := io.ReadFull(crand.Reader, data) - OK(t, err) - id := backend.Hash(data) + length := rand.Intn(1<<23) + 200000 + data := Random(23, length) + // use the first 32 byte as the ID + copy(id[:], data) h := backend.Handle{ Type: backend.Data, Name: fmt.Sprintf("%s-%d", id, i), } - err = b.Save(h, data) + err := b.Save(h, data) OK(t, err) buf, err := backend.LoadAll(b, h, nil) diff --git a/crypto/crypto_test.go b/crypto/crypto_test.go index a821e105f..311eb1835 100644 --- a/crypto/crypto_test.go +++ b/crypto/crypto_test.go @@ -23,10 +23,7 @@ func TestEncryptDecrypt(t *testing.T) { } for _, size := range tests { - data := make([]byte, size) - _, err := io.ReadFull(RandomReader(42, size), data) - OK(t, err) - + data := Random(42, size) buf := make([]byte, size+crypto.Extension) ciphertext, err := crypto.Encrypt(k, buf, data) @@ -140,7 +137,7 @@ func BenchmarkEncryptWriter(b *testing.B) { b.SetBytes(int64(size)) for i := 0; i < b.N; i++ { - rd := RandomReader(23, size) + rd := RandomLimitReader(23, size) wr := crypto.EncryptTo(k, ioutil.Discard) n, err := io.Copy(wr, rd) OK(b, err) @@ -200,7 +197,7 @@ func BenchmarkEncryptDecryptReader(b *testing.B) { buf := bytes.NewBuffer(nil) for i := 0; i < b.N; i++ { - rd := RandomReader(23, size) + rd := RandomLimitReader(23, size) buf.Reset() wr := crypto.EncryptTo(k, buf) _, err := io.Copy(wr, rd) @@ -245,14 +242,12 @@ func TestEncryptStreamWriter(t *testing.T) { } for _, size := range tests { - data := make([]byte, size) - _, err := io.ReadFull(RandomReader(42, size), data) - OK(t, err) + data := Random(42, size) ciphertext := bytes.NewBuffer(nil) wr := crypto.EncryptTo(k, ciphertext) - _, err = io.Copy(wr, bytes.NewReader(data)) + _, err := io.Copy(wr, bytes.NewReader(data)) OK(t, err) OK(t, wr.Close()) @@ -279,10 +274,8 @@ func TestDecryptStreamReader(t *testing.T) { } for _, size := range tests { - data := make([]byte, size) - _, err := io.ReadFull(RandomReader(42, size), data) - OK(t, err) - + data := Random(42, size) + var err error ciphertext := make([]byte, size+crypto.Extension) // encrypt with default function @@ -313,14 +306,12 @@ func TestEncryptWriter(t *testing.T) { } for _, size := range tests { - data := make([]byte, size) - _, err := io.ReadFull(RandomReader(42, size), data) - OK(t, err) + data := Random(42, size) buf := bytes.NewBuffer(nil) wr := crypto.EncryptTo(k, buf) - _, err = io.Copy(wr, bytes.NewReader(data)) + _, err := io.Copy(wr, bytes.NewReader(data)) OK(t, err) OK(t, wr.Close()) diff --git a/test/helpers.go b/test/helpers.go index 95a1f003c..636dddd8f 100644 --- a/test/helpers.go +++ b/test/helpers.go @@ -75,14 +75,33 @@ func ParseID(s string) backend.ID { // Random returns size bytes of pseudo-random data derived from the seed. func Random(seed, count int) []byte { - buf := make([]byte, count) + p := make([]byte, count) rnd := mrand.New(mrand.NewSource(int64(seed))) - for i := 0; i < count; i++ { - buf[i] = byte(rnd.Uint32()) + + for i := 0; i < len(p); i += 8 { + val := rnd.Int63() + var data = []byte{ + byte((val >> 0) & 0xff), + byte((val >> 8) & 0xff), + byte((val >> 16) & 0xff), + byte((val >> 24) & 0xff), + byte((val >> 32) & 0xff), + byte((val >> 40) & 0xff), + byte((val >> 48) & 0xff), + byte((val >> 56) & 0xff), + } + + for j := range data { + cur := i + j + if len(p) >= cur { + break + } + p[cur] = data[j] + } } - return buf + return p } type rndReader struct { @@ -90,18 +109,41 @@ type rndReader struct { } func (r *rndReader) Read(p []byte) (int, error) { - for i := range p { - p[i] = byte(r.src.Uint32()) + for i := 0; i < len(p); i += 8 { + val := r.src.Int63() + var data = []byte{ + byte((val >> 0) & 0xff), + byte((val >> 8) & 0xff), + byte((val >> 16) & 0xff), + byte((val >> 24) & 0xff), + byte((val >> 32) & 0xff), + byte((val >> 40) & 0xff), + byte((val >> 48) & 0xff), + byte((val >> 56) & 0xff), + } + + for j := range data { + cur := i + j + if len(p) >= cur { + break + } + p[cur] = data[j] + } } return len(p), nil } -// RandomReader returns a reader that returns size bytes of pseudo-random data +// RandomReader returns a reader that returns deterministic pseudo-random data // derived from the seed. -func RandomReader(seed, size int) io.Reader { - r := &rndReader{src: mrand.New(mrand.NewSource(int64(seed)))} - return io.LimitReader(r, int64(size)) +func RandomReader(seed int) io.Reader { + return &rndReader{src: mrand.New(mrand.NewSource(int64(seed)))} +} + +// RandomLimitReader returns a reader that returns size bytes of deterministic +// pseudo-random data derived from the seed. +func RandomLimitReader(seed, size int) io.Reader { + return io.LimitReader(RandomReader(seed), int64(size)) } // GenRandom returns a []byte filled with up to 1000 random bytes. From fe565e17c38f43b064a65dd8235747e1da0d3a7c Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 17:52:44 +0100 Subject: [PATCH 31/56] Key: Use Save() instead of Create() --- repository/key.go | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/repository/key.go b/repository/key.go index 78665513b..c44b43710 100644 --- a/repository/key.go +++ b/repository/key.go @@ -2,8 +2,6 @@ package repository import ( "crypto/rand" - "crypto/sha256" - "encoding/hex" "encoding/json" "errors" "fmt" @@ -191,26 +189,17 @@ func AddKey(s *Repository, password string, template *crypto.Key) (*Key, error) } // store in repository and return - blob, err := s.be.Create() + h := backend.Handle{ + Type: backend.Key, + Name: backend.Hash(buf).String(), + } + + err = s.be.Save(h, buf) if err != nil { return nil, err } - plainhw := backend.NewHashingWriter(blob, sha256.New()) - - _, err = plainhw.Write(buf) - if err != nil { - return nil, err - } - - name := hex.EncodeToString(plainhw.Sum(nil)) - - err = blob.Finalize(backend.Key, name) - if err != nil { - return nil, err - } - - newkey.name = name + newkey.name = h.Name return newkey, nil } From 35f9eae6c376d3e2bbafaa721a6842fb01d3505c Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 18:01:00 +0100 Subject: [PATCH 32/56] local backend: do not call Sync() on directory This fails at least on Windows. --- backend/local/local.go | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/backend/local/local.go b/backend/local/local.go index 5f836a8a5..e45221f91 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -283,23 +283,7 @@ func (b *Local) Save(h backend.Handle, p []byte) (err error) { return err } - err = setNewFileMode(f, fi) - if err != nil { - return err - } - - // try to flush directory - d, err := os.Open(filepath.Dir(f)) - if err != nil { - return err - } - - err = d.Sync() - if err != nil { - return err - } - - return d.Close() + return setNewFileMode(f, fi) } // Stat returns information about a blob. From 01e40e62bf82308d676cdf003c3059fb0c211c53 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 18:50:41 +0100 Subject: [PATCH 33/56] repo: Use Save() instead of Create() --- repository/repository.go | 35 ++++++++++------------------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/repository/repository.go b/repository/repository.go index a3ed9c60f..275d6fdee 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -265,44 +265,29 @@ func (r *Repository) SaveJSON(t pack.BlobType, item interface{}) (backend.ID, er // SaveJSONUnpacked serialises item as JSON and encrypts and saves it in the // backend as type t, without a pack. It returns the storage hash. func (r *Repository) SaveJSONUnpacked(t backend.Type, item interface{}) (backend.ID, error) { - // create file - blob, err := r.be.Create() - if err != nil { - return backend.ID{}, err - } - debug.Log("Repo.SaveJSONUnpacked", "create new blob %v", t) - - // hash - hw := backend.NewHashingWriter(blob, sha256.New()) - - // encrypt blob - ewr := crypto.EncryptTo(r.key, hw) - - enc := json.NewEncoder(ewr) - err = enc.Encode(item) + debug.Log("Repo.SaveJSONUnpacked", "save new blob %v", t) + plaintext, err := json.Marshal(item) if err != nil { return backend.ID{}, fmt.Errorf("json.Encode: %v", err) } - err = ewr.Close() + ciphertext := make([]byte, len(plaintext)+crypto.Extension) + ciphertext, err = crypto.Encrypt(r.key, ciphertext, plaintext) if err != nil { return backend.ID{}, err } - // finalize blob in the backend - hash := hw.Sum(nil) - sid := backend.ID{} - copy(sid[:], hash) + id := backend.Hash(ciphertext) + h := backend.Handle{Type: t, Name: id.String()} - err = blob.Finalize(t, sid.String()) + err = r.be.Save(h, ciphertext) if err != nil { - debug.Log("Repo.SaveJSONUnpacked", "error saving blob %v as %v: %v", t, sid, err) + debug.Log("Repo.SaveJSONUnpacked", "error saving blob %v: %v", h, err) return backend.ID{}, err } - debug.Log("Repo.SaveJSONUnpacked", "new blob %v saved as %v", t, sid) - - return sid, nil + debug.Log("Repo.SaveJSONUnpacked", "blob %v saved", h) + return id, nil } // Flush saves all remaining packs. From cfdd3a853de8aca7505854a5a00082baf60252f7 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 18:52:11 +0100 Subject: [PATCH 34/56] Remove usage of CreateEncryptedBlob() --- repository/index.go | 25 +++---------------------- repository/repository.go | 32 ++++++++++++++------------------ 2 files changed, 17 insertions(+), 40 deletions(-) diff --git a/repository/index.go b/repository/index.go index 093667336..87a53c8ac 100644 --- a/repository/index.go +++ b/repository/index.go @@ -594,33 +594,14 @@ func ConvertIndex(repo *Repository, id backend.ID) (backend.ID, error) { return id, err } - blob, err := repo.CreateEncryptedBlob(backend.Index) - if err != nil { - return id, err - } - + buf := bytes.NewBuffer(nil) idx.supersedes = backend.IDs{id} - err = idx.Encode(blob) + err = idx.Encode(buf) if err != nil { debug.Log("ConvertIndex", "oldIdx.Encode() returned error: %v", err) return id, err } - err = blob.Close() - if err != nil { - debug.Log("ConvertIndex", "blob.Close() returned error: %v", err) - return id, err - } - - newID := blob.ID() - debug.Log("ConvertIndex", "index %v converted to new format as %v", id.Str(), newID.Str()) - - err = repo.be.Remove(backend.Index, id.String()) - if err != nil { - debug.Log("ConvertIndex", "backend.Remove(%v) returned error: %v", id.Str(), err) - return id, err - } - - return newID, nil + return repo.SaveUnpacked(backend.Index, buf.Bytes()) } diff --git a/repository/repository.go b/repository/repository.go index 275d6fdee..1469bee3d 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -271,13 +271,19 @@ func (r *Repository) SaveJSONUnpacked(t backend.Type, item interface{}) (backend return backend.ID{}, fmt.Errorf("json.Encode: %v", err) } - ciphertext := make([]byte, len(plaintext)+crypto.Extension) - ciphertext, err = crypto.Encrypt(r.key, ciphertext, plaintext) + return r.SaveUnpacked(t, plaintext) +} + +// SaveUnpacked encrypts data and stores it in the backend. Returned is the +// storage hash. +func (r *Repository) SaveUnpacked(t backend.Type, p []byte) (id backend.ID, err error) { + ciphertext := make([]byte, len(p)+crypto.Extension) + ciphertext, err = r.Encrypt(ciphertext, p) if err != nil { return backend.ID{}, err } - id := backend.Hash(ciphertext) + id = backend.Hash(ciphertext) h := backend.Handle{Type: t, Name: id.String()} err = r.be.Save(h, ciphertext) @@ -377,26 +383,16 @@ func (bw *BlobWriter) ID() backend.ID { return bw.id } -// SaveIndex saves an index to repo's backend. +// SaveIndex saves an index in the repository. func SaveIndex(repo *Repository, index *Index) (backend.ID, error) { - blob, err := repo.CreateEncryptedBlob(backend.Index) + buf := bytes.NewBuffer(nil) + + err := index.Finalize(buf) if err != nil { return backend.ID{}, err } - err = index.Finalize(blob) - if err != nil { - return backend.ID{}, err - } - - err = blob.Close() - if err != nil { - return backend.ID{}, err - } - - sid := blob.ID() - err = index.SetID(sid) - return sid, err + return repo.SaveUnpacked(backend.Index, buf.Bytes()) } // saveIndex saves all indexes in the backend. From ac2fe4e04ff63d9aa8fdab1bf6fbe059d444889f Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 18:53:39 +0100 Subject: [PATCH 35/56] Remove BlobWriter --- repository/repository.go | 55 ---------------------------------------- 1 file changed, 55 deletions(-) diff --git a/repository/repository.go b/repository/repository.go index 1469bee3d..88f289338 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -2,7 +2,6 @@ package repository import ( "bytes" - "crypto/sha256" "encoding/json" "errors" "fmt" @@ -329,60 +328,6 @@ func (r *Repository) SetIndex(i *MasterIndex) { r.idx = i } -// BlobWriter encrypts and saves the data written to it in a backend. After -// Close() was called, ID() returns the backend.ID. -type BlobWriter struct { - id backend.ID - blob backend.Blob - hw *backend.HashingWriter - ewr io.WriteCloser - t backend.Type - closed bool -} - -// CreateEncryptedBlob returns a BlobWriter that encrypts and saves the data -// written to it in the backend. After Close() was called, ID() returns the -// backend.ID. -func (r *Repository) CreateEncryptedBlob(t backend.Type) (*BlobWriter, error) { - blob, err := r.be.Create() - if err != nil { - return nil, err - } - - // hash - hw := backend.NewHashingWriter(blob, sha256.New()) - - // encrypt blob - ewr := crypto.EncryptTo(r.key, hw) - - return &BlobWriter{t: t, blob: blob, hw: hw, ewr: ewr}, nil -} - -func (bw *BlobWriter) Write(buf []byte) (int, error) { - return bw.ewr.Write(buf) -} - -// Close finalizes the blob in the backend, afterwards ID() can be used to retrieve the ID. -func (bw *BlobWriter) Close() error { - if bw.closed { - return errors.New("BlobWriter already closed") - } - bw.closed = true - - err := bw.ewr.Close() - if err != nil { - return err - } - - copy(bw.id[:], bw.hw.Sum(nil)) - return bw.blob.Finalize(bw.t, bw.id.String()) -} - -// ID returns the Id the blob has been written to after Close() was called. -func (bw *BlobWriter) ID() backend.ID { - return bw.id -} - // SaveIndex saves an index in the repository. func SaveIndex(repo *Repository, index *Index) (backend.ID, error) { buf := bytes.NewBuffer(nil) From 1a95e48389e16a34b95ed269c1d1dcbc0ca0c839 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 18:58:15 +0100 Subject: [PATCH 36/56] Remove unneeded special readers --- backend/reader.go | 73 ------------------------------------- backend/reader_test.go | 81 ------------------------------------------ 2 files changed, 154 deletions(-) delete mode 100644 backend/reader.go delete mode 100644 backend/reader_test.go diff --git a/backend/reader.go b/backend/reader.go deleted file mode 100644 index eabe9527b..000000000 --- a/backend/reader.go +++ /dev/null @@ -1,73 +0,0 @@ -package backend - -import ( - "hash" - "io" -) - -type HashAppendReader struct { - r io.Reader - h hash.Hash - sum []byte - closed bool -} - -func NewHashAppendReader(r io.Reader, h hash.Hash) *HashAppendReader { - return &HashAppendReader{ - h: h, - r: io.TeeReader(r, h), - sum: make([]byte, 0, h.Size()), - } -} - -func (h *HashAppendReader) Read(p []byte) (n int, err error) { - if !h.closed { - n, err = h.r.Read(p) - - if err == io.EOF { - h.closed = true - h.sum = h.h.Sum(h.sum) - } else if err != nil { - return - } - } - - if h.closed { - // output hash - r := len(p) - n - - if r > 0 { - c := copy(p[n:], h.sum) - h.sum = h.sum[c:] - - n += c - err = nil - } - - if len(h.sum) == 0 { - err = io.EOF - } - } - - return -} - -type HashingReader struct { - r io.Reader - h hash.Hash -} - -func NewHashingReader(r io.Reader, h hash.Hash) *HashingReader { - return &HashingReader{ - h: h, - r: io.TeeReader(r, h), - } -} - -func (h *HashingReader) Read(p []byte) (int, error) { - return h.r.Read(p) -} - -func (h *HashingReader) Sum(d []byte) []byte { - return h.h.Sum(d) -} diff --git a/backend/reader_test.go b/backend/reader_test.go deleted file mode 100644 index b4a23eaea..000000000 --- a/backend/reader_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package backend_test - -import ( - "bytes" - "crypto/rand" - "crypto/sha256" - "io" - "io/ioutil" - "testing" - - "github.com/restic/restic/backend" - . "github.com/restic/restic/test" -) - -func TestHashAppendReader(t *testing.T) { - tests := []int{5, 23, 2<<18 + 23, 1 << 20} - - for _, size := range tests { - data := make([]byte, size) - _, err := io.ReadFull(rand.Reader, data) - if err != nil { - t.Fatalf("ReadFull: %v", err) - } - - expectedHash := sha256.Sum256(data) - - rd := backend.NewHashAppendReader(bytes.NewReader(data), sha256.New()) - - target := bytes.NewBuffer(nil) - n, err := io.Copy(target, rd) - OK(t, err) - - Assert(t, n == int64(size)+int64(len(expectedHash)), - "HashAppendReader: invalid number of bytes read: got %d, expected %d", - n, size+len(expectedHash)) - - r := target.Bytes() - resultingHash := r[len(r)-len(expectedHash):] - Assert(t, bytes.Equal(expectedHash[:], resultingHash), - "HashAppendReader: hashes do not match: expected %02x, got %02x", - expectedHash, resultingHash) - - // try to read again, must return io.EOF - n2, err := rd.Read(make([]byte, 100)) - Assert(t, n2 == 0, "HashAppendReader returned %d additional bytes", n) - Assert(t, err == io.EOF, "HashAppendReader returned %v instead of EOF", err) - } -} - -func TestHashingReader(t *testing.T) { - tests := []int{5, 23, 2<<18 + 23, 1 << 20} - - for _, size := range tests { - data := make([]byte, size) - _, err := io.ReadFull(rand.Reader, data) - if err != nil { - t.Fatalf("ReadFull: %v", err) - } - - expectedHash := sha256.Sum256(data) - - rd := backend.NewHashingReader(bytes.NewReader(data), sha256.New()) - - n, err := io.Copy(ioutil.Discard, rd) - OK(t, err) - - Assert(t, n == int64(size), - "HashAppendReader: invalid number of bytes read: got %d, expected %d", - n, size) - - resultingHash := rd.Sum(nil) - Assert(t, bytes.Equal(expectedHash[:], resultingHash), - "HashAppendReader: hashes do not match: expected %02x, got %02x", - expectedHash, resultingHash) - - // try to read again, must return io.EOF - n2, err := rd.Read(make([]byte, 100)) - Assert(t, n2 == 0, "HashAppendReader returned %d additional bytes", n) - Assert(t, err == io.EOF, "HashAppendReader returned %v instead of EOF", err) - } -} From ea29ad6f96f897517259557cccf13cc315c3db49 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 19:30:14 +0100 Subject: [PATCH 37/56] Remove last ocurrence of Create() --- pack/pack.go | 76 +++++++++++++++--------------------- pack/pack_test.go | 12 ++---- repository/packer_manager.go | 25 ++++++------ 3 files changed, 47 insertions(+), 66 deletions(-) diff --git a/pack/pack.go b/pack/pack.go index 697adb8aa..8a96a942d 100644 --- a/pack/pack.go +++ b/pack/pack.go @@ -1,7 +1,7 @@ package pack import ( - "crypto/sha256" + "bytes" "encoding/binary" "errors" "fmt" @@ -12,8 +12,10 @@ import ( "github.com/restic/restic/crypto" ) +// BlobType specifies what a blob stored in a pack is. type BlobType uint8 +// These are the blob types that can be stored in a pack. const ( Data BlobType = 0 Tree = 1 @@ -30,6 +32,7 @@ func (t BlobType) String() string { return fmt.Sprintf("", t) } +// MarshalJSON encodes the BlobType into JSON. func (t BlobType) MarshalJSON() ([]byte, error) { switch t { case Data: @@ -41,6 +44,7 @@ func (t BlobType) MarshalJSON() ([]byte, error) { return nil, errors.New("unknown blob type") } +// UnmarshalJSON decodes the BlobType from JSON. func (t *BlobType) UnmarshalJSON(buf []byte) error { switch string(buf) { case `"data"`: @@ -79,16 +83,15 @@ type Packer struct { bytes uint k *crypto.Key - wr io.Writer - hw *backend.HashingWriter + buf *bytes.Buffer m sync.Mutex } // NewPacker returns a new Packer that can be used to pack blobs // together. -func NewPacker(k *crypto.Key, w io.Writer) *Packer { - return &Packer{k: k, wr: w, hw: backend.NewHashingWriter(w, sha256.New())} +func NewPacker(k *crypto.Key, buf []byte) *Packer { + return &Packer{k: k, buf: bytes.NewBuffer(buf)} } // Add saves the data read from rd as a new blob to the packer. Returned is the @@ -99,7 +102,7 @@ func (p *Packer) Add(t BlobType, id backend.ID, rd io.Reader) (int64, error) { c := Blob{Type: t, ID: id} - n, err := io.Copy(p.hw, rd) + n, err := io.Copy(p.buf, rd) c.Length = uint(n) c.Offset = p.bytes p.bytes += uint(n) @@ -118,45 +121,47 @@ type headerEntry struct { } // Finalize writes the header for all added blobs and finalizes the pack. -// Returned are the complete number of bytes written, including the header. -// After Finalize() has finished, the ID of this pack can be obtained by -// calling ID(). -func (p *Packer) Finalize() (bytesWritten uint, err error) { +// Returned are all bytes written, including the header. +func (p *Packer) Finalize() ([]byte, error) { p.m.Lock() defer p.m.Unlock() - bytesWritten = p.bytes + bytesWritten := p.bytes - // create writer to encrypt header - wr := crypto.EncryptTo(p.k, p.hw) - - bytesHeader, err := p.writeHeader(wr) + hdrBuf := bytes.NewBuffer(nil) + bytesHeader, err := p.writeHeader(hdrBuf) if err != nil { - wr.Close() - return bytesWritten + bytesHeader, err + return nil, err } - bytesWritten += bytesHeader - - // finalize encrypted header - err = wr.Close() + encryptedHeader, err := crypto.Encrypt(p.k, nil, hdrBuf.Bytes()) if err != nil { - return bytesWritten, err + return nil, err } - // account for crypto overhead - bytesWritten += crypto.Extension + // append the header + n, err := p.buf.Write(encryptedHeader) + if err != nil { + return nil, err + } + + hdrBytes := bytesHeader + crypto.Extension + if uint(n) != hdrBytes { + return nil, errors.New("wrong number of bytes written") + } + + bytesWritten += hdrBytes // write length - err = binary.Write(p.hw, binary.LittleEndian, uint32(uint(len(p.blobs))*entrySize+crypto.Extension)) + err = binary.Write(p.buf, binary.LittleEndian, uint32(uint(len(p.blobs))*entrySize+crypto.Extension)) if err != nil { - return bytesWritten, err + return nil, err } bytesWritten += uint(binary.Size(uint32(0))) p.bytes = uint(bytesWritten) - return bytesWritten, nil + return p.buf.Bytes(), nil } // writeHeader constructs and writes the header to wr. @@ -179,18 +184,6 @@ func (p *Packer) writeHeader(wr io.Writer) (bytesWritten uint, err error) { return } -// ID returns the ID of all data written so far. -func (p *Packer) ID() backend.ID { - p.m.Lock() - defer p.m.Unlock() - - hash := p.hw.Sum(nil) - id := backend.ID{} - copy(id[:], hash) - - return id -} - // Size returns the number of bytes written so far. func (p *Packer) Size() uint { p.m.Lock() @@ -215,11 +208,6 @@ func (p *Packer) Blobs() []Blob { return p.blobs } -// Writer returns the underlying writer. -func (p *Packer) Writer() io.Writer { - return p.wr -} - func (p *Packer) String() string { return fmt.Sprintf("", len(p.blobs), p.bytes) } diff --git a/pack/pack_test.go b/pack/pack_test.go index 28ef4c22c..0d5c1f155 100644 --- a/pack/pack_test.go +++ b/pack/pack_test.go @@ -34,23 +34,19 @@ func TestCreatePack(t *testing.T) { bufs = append(bufs, Buf{data: b, id: h}) } - file := bytes.NewBuffer(nil) - // create random keys k := crypto.NewRandomKey() // pack blobs - p := pack.NewPacker(k, file) + p := pack.NewPacker(k, nil) for _, b := range bufs { p.Add(pack.Tree, b.id, bytes.NewReader(b.data)) } - // write file - n, err := p.Finalize() + packData, err := p.Finalize() OK(t, err) written := 0 - // data for _, l := range lengths { written += l } @@ -62,11 +58,11 @@ func TestCreatePack(t *testing.T) { written += crypto.Extension // check length - Equals(t, uint(written), n) + Equals(t, written, len(packData)) Equals(t, uint(written), p.Size()) // read and parse it again - rd := bytes.NewReader(file.Bytes()) + rd := bytes.NewReader(packData) np, err := pack.NewUnpacker(k, rd) OK(t, err) Equals(t, len(np.Entries), len(bufs)) diff --git a/repository/packer_manager.go b/repository/packer_manager.go index 99b74cea4..42ffe96cb 100644 --- a/repository/packer_manager.go +++ b/repository/packer_manager.go @@ -42,12 +42,8 @@ func (r *packerManager) findPacker(size uint) (*pack.Packer, error) { } // no suitable packer found, return new - blob, err := r.be.Create() - if err != nil { - return nil, err - } - debug.Log("Repo.findPacker", "create new pack %p for %d bytes", blob, size) - return pack.NewPacker(r.key, blob), nil + debug.Log("Repo.findPacker", "create new pack for %d bytes", size) + return pack.NewPacker(r.key, nil), nil } // insertPacker appends p to s.packs. @@ -62,28 +58,29 @@ func (r *packerManager) insertPacker(p *pack.Packer) { // savePacker stores p in the backend. func (r *Repository) savePacker(p *pack.Packer) error { debug.Log("Repo.savePacker", "save packer with %d blobs\n", p.Count()) - _, err := p.Finalize() + data, err := p.Finalize() if err != nil { return err } - // move file to the final location - sid := p.ID() - err = p.Writer().(backend.Blob).Finalize(backend.Data, sid.String()) + id := backend.Hash(data) + h := backend.Handle{Type: backend.Data, Name: id.String()} + + err = r.be.Save(h, data) if err != nil { - debug.Log("Repo.savePacker", "blob Finalize() error: %v", err) + debug.Log("Repo.savePacker", "Save(%v) error: %v", h, err) return err } - debug.Log("Repo.savePacker", "saved as %v", sid.Str()) + debug.Log("Repo.savePacker", "saved as %v", h) // update blobs in the index for _, b := range p.Blobs() { - debug.Log("Repo.savePacker", " updating blob %v to pack %v", b.ID.Str(), sid.Str()) + debug.Log("Repo.savePacker", " updating blob %v to pack %v", b.ID.Str(), id.Str()) r.idx.Current().Store(PackedBlob{ Type: b.Type, ID: b.ID, - PackID: sid, + PackID: id, Offset: b.Offset, Length: uint(b.Length), }) From 1547d3b656e7ec42103dbed4a2eb35fe75a8be72 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 20:23:50 +0100 Subject: [PATCH 38/56] Remove Create() everywhere --- backend/interface.go | 17 ------ backend/local/backend_test.go | 7 --- backend/local/local.go | 90 +---------------------------- backend/mem/backend_test.go | 7 --- backend/mem/mem_backend.go | 14 ++--- backend/mock_backend.go | 9 --- backend/s3/backend_test.go | 7 --- backend/s3/s3.go | 79 +------------------------ backend/sftp/backend_test.go | 7 --- backend/sftp/sftp.go | 70 +++-------------------- backend/test/backend_test.go | 7 --- backend/test/tests.go | 105 ++++------------------------------ backend/utils_test.go | 7 +-- 13 files changed, 25 insertions(+), 401 deletions(-) diff --git a/backend/interface.go b/backend/interface.go index 9fff4bf1e..fb0927c6e 100644 --- a/backend/interface.go +++ b/backend/interface.go @@ -1,7 +1,5 @@ package backend -import "io" - // Type is the type of a Blob. type Type string @@ -21,10 +19,6 @@ type Backend interface { // repository. Location() string - // Create creates a new Blob. The data is available only after Finalize() - // has been called on the returned Blob. - Create() (Blob, error) - // Test a boolean value whether a Blob with the name and type exists. Test(t Type, name string) (bool, error) @@ -65,14 +59,3 @@ type Deleter interface { type BlobInfo struct { Size int64 } - -// Blob is old. -type Blob interface { - io.Writer - - // Finalize moves the data blob to the final location for type and name. - Finalize(t Type, name string) error - - // Size returns the number of bytes written to the backend so far. - Size() uint -} diff --git a/backend/local/backend_test.go b/backend/local/backend_test.go index 5617ab74a..b2d5e7b0f 100644 --- a/backend/local/backend_test.go +++ b/backend/local/backend_test.go @@ -51,13 +51,6 @@ func TestLocalBackendLoad(t *testing.T) { test.TestLoad(t) } -func TestLocalBackendWrite(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestWrite(t) -} - func TestLocalBackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/local/local.go b/backend/local/local.go index e45221f91..9a6409a75 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -14,8 +14,7 @@ import ( "github.com/restic/restic/debug" ) -var ErrWrongData = errors.New("wrong data returned by backend, checksum does not match") - +// Local is a backend in a local directory. type Local struct { p string mu sync.Mutex @@ -80,93 +79,6 @@ func (b *Local) Location() string { return b.p } -// Return temp directory in correct directory for this backend. -func (b *Local) tempFile() (*os.File, error) { - return ioutil.TempFile(filepath.Join(b.p, backend.Paths.Temp), "temp-") -} - -type localBlob struct { - f *os.File - size uint - final bool - basedir string -} - -func (lb *localBlob) Write(p []byte) (int, error) { - if lb.final { - return 0, errors.New("blob already closed") - } - - n, err := lb.f.Write(p) - lb.size += uint(n) - return n, err -} - -func (lb *localBlob) Size() uint { - return lb.size -} - -func (lb *localBlob) Finalize(t backend.Type, name string) error { - if lb.final { - return errors.New("Already finalized") - } - - lb.final = true - - err := lb.f.Close() - if err != nil { - return fmt.Errorf("local: file.Close: %v", err) - } - - f := filename(lb.basedir, t, name) - - // create directories if necessary, ignore errors - if t == backend.Data { - os.MkdirAll(filepath.Dir(f), backend.Modes.Dir) - } - - // test if new path already exists - if _, err := os.Stat(f); err == nil { - return fmt.Errorf("Close(): file %v already exists", f) - } - - if err := os.Rename(lb.f.Name(), f); err != nil { - return err - } - - // set mode to read-only - fi, err := os.Stat(f) - if err != nil { - return err - } - - return setNewFileMode(f, fi) -} - -// Create creates a new Blob. The data is available only after Finalize() -// has been called on the returned Blob. -func (b *Local) Create() (backend.Blob, error) { - // TODO: make sure that tempfile is removed upon error - - // create tempfile in backend - file, err := b.tempFile() - if err != nil { - return nil, err - } - - blob := localBlob{ - f: file, - basedir: b.p, - } - - b.mu.Lock() - open, _ := b.open["blobs"] - b.open["blobs"] = append(open, file) - b.mu.Unlock() - - return &blob, nil -} - // Construct path for given Type and name. func filename(base string, t backend.Type, name string) string { if t == backend.Config { diff --git a/backend/mem/backend_test.go b/backend/mem/backend_test.go index 9b1a602a1..31f86e4fc 100644 --- a/backend/mem/backend_test.go +++ b/backend/mem/backend_test.go @@ -51,13 +51,6 @@ func TestMemBackendLoad(t *testing.T) { test.TestLoad(t) } -func TestMemBackendWrite(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestWrite(t) -} - func TestMemBackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/mem/mem_backend.go b/backend/mem/mem_backend.go index 91831dc5e..4de0afe82 100644 --- a/backend/mem/mem_backend.go +++ b/backend/mem/mem_backend.go @@ -37,10 +37,6 @@ func New() *MemoryBackend { return memTest(be, t, name) } - be.MockBackend.CreateFn = func() (backend.Blob, error) { - return memCreate(be) - } - be.MockBackend.LoadFn = func(h backend.Handle, p []byte, off int64) (int, error) { return memLoad(be, h, p, off) } @@ -127,12 +123,6 @@ func (e *tempMemEntry) Finalize(t backend.Type, name string) error { return e.be.insert(t, name, e.data.Bytes()) } -func memCreate(be *MemoryBackend) (backend.Blob, error) { - blob := &tempMemEntry{be: be} - debug.Log("MemoryBackend.Create", "create new blob %p", blob) - return blob, nil -} - func memLoad(be *MemoryBackend, h backend.Handle, p []byte, off int64) (int, error) { if err := h.Valid(); err != nil { return 0, err @@ -179,6 +169,10 @@ func memSave(be *MemoryBackend, h backend.Handle, p []byte) error { h.Name = "" } + if _, ok := be.data[entry{h.Type, h.Name}]; ok { + return errors.New("file already exists") + } + debug.Log("MemoryBackend.Save", "save %v bytes at %v", len(p), h) buf := make([]byte, len(p)) copy(buf, p) diff --git a/backend/mock_backend.go b/backend/mock_backend.go index 9e692883f..e685ba556 100644 --- a/backend/mock_backend.go +++ b/backend/mock_backend.go @@ -6,7 +6,6 @@ import "errors" // should only be used for tests. type MockBackend struct { CloseFn func() error - CreateFn func() (Blob, error) LoadFn func(h Handle, p []byte, off int64) (int, error) SaveFn func(h Handle, p []byte) error StatFn func(h Handle) (BlobInfo, error) @@ -33,14 +32,6 @@ func (m *MockBackend) Location() string { return m.LocationFn() } -func (m *MockBackend) Create() (Blob, error) { - if m.CreateFn == nil { - return nil, errors.New("not implemented") - } - - return m.CreateFn() -} - func (m *MockBackend) Load(h Handle, p []byte, off int64) (int, error) { if m.LoadFn == nil { return 0, errors.New("not implemented") diff --git a/backend/s3/backend_test.go b/backend/s3/backend_test.go index 4bd4d6bee..8db0a31f9 100644 --- a/backend/s3/backend_test.go +++ b/backend/s3/backend_test.go @@ -51,13 +51,6 @@ func TestS3BackendLoad(t *testing.T) { test.TestLoad(t) } -func TestS3BackendWrite(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestWrite(t) -} - func TestS3BackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 9fd5c6152..a8f69c4c9 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -2,7 +2,6 @@ package s3 import ( "bytes" - "errors" "io" "strings" @@ -23,6 +22,7 @@ func s3path(t backend.Type, name string) string { return backendPrefix + "/" + string(t) + "/" + name } +// S3Backend is a backend which stores the data on an S3 endpoint. type S3Backend struct { client minio.CloudStorageClient connChan chan struct{} @@ -68,83 +68,6 @@ func (be *S3Backend) Location() string { return be.bucketname } -type s3Blob struct { - b *S3Backend - buf *bytes.Buffer - final bool -} - -func (bb *s3Blob) Write(p []byte) (int, error) { - if bb.final { - return 0, errors.New("blob already closed") - } - - n, err := bb.buf.Write(p) - return n, err -} - -func (bb *s3Blob) Read(p []byte) (int, error) { - return bb.buf.Read(p) -} - -func (bb *s3Blob) Close() error { - bb.final = true - bb.buf.Reset() - return nil -} - -func (bb *s3Blob) Size() uint { - return uint(bb.buf.Len()) -} - -func (bb *s3Blob) Finalize(t backend.Type, name string) error { - debug.Log("s3.blob.Finalize()", "bucket %v, finalize %v, %d bytes", bb.b.bucketname, name, bb.buf.Len()) - if bb.final { - return errors.New("Already finalized") - } - - bb.final = true - - path := s3path(t, name) - - // Check key does not already exist - _, err := bb.b.client.StatObject(bb.b.bucketname, path) - if err == nil { - debug.Log("s3.blob.Finalize()", "%v already exists", name) - return errors.New("key already exists") - } - - expectedBytes := bb.buf.Len() - - <-bb.b.connChan - debug.Log("s3.Finalize", "PutObject(%v, %v, %v, %v)", - bb.b.bucketname, path, int64(bb.buf.Len()), "binary/octet-stream") - n, err := bb.b.client.PutObject(bb.b.bucketname, path, bb.buf, "binary/octet-stream") - debug.Log("s3.Finalize", "finalized %v -> n %v, err %#v", path, n, err) - bb.b.connChan <- struct{}{} - - if err != nil { - return err - } - - if n != int64(expectedBytes) { - return errors.New("could not store all bytes") - } - - return nil -} - -// Create creates a new Blob. The data is available only after Finalize() -// has been called on the returned Blob. -func (be *S3Backend) Create() (backend.Blob, error) { - blob := s3Blob{ - b: be, - buf: &bytes.Buffer{}, - } - - return &blob, nil -} - // Load returns the data stored in the backend for h at the given offset // and saves it in p. Load has the same semantics as io.ReaderAt. func (be S3Backend) Load(h backend.Handle, p []byte, off int64) (int, error) { diff --git a/backend/sftp/backend_test.go b/backend/sftp/backend_test.go index ca268b3a3..afab17e60 100644 --- a/backend/sftp/backend_test.go +++ b/backend/sftp/backend_test.go @@ -51,13 +51,6 @@ func TestSftpBackendLoad(t *testing.T) { test.TestLoad(t) } -func TestSftpBackendWrite(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestWrite(t) -} - func TestSftpBackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index a7ad4955e..94cc91132 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -22,6 +22,7 @@ const ( tempfileRandomSuffixLength = 10 ) +// SFTP is a backend in a directory accessed via SFTP. type SFTP struct { c *sftp.Client p string @@ -253,64 +254,7 @@ func (r *SFTP) renameFile(oldname string, t backend.Type, name string) error { return r.c.Chmod(filename, fi.Mode()&os.FileMode(^uint32(0222))) } -type sftpBlob struct { - f *sftp.File - tempname string - size uint - closed bool - backend *SFTP -} - -func (sb *sftpBlob) Finalize(t backend.Type, name string) error { - if sb.closed { - return errors.New("Close() called on closed file") - } - sb.closed = true - - err := sb.f.Close() - if err != nil { - return fmt.Errorf("sftp: file.Close: %v", err) - } - - // rename file - err = sb.backend.renameFile(sb.tempname, t, name) - if err != nil { - return fmt.Errorf("sftp: renameFile: %v", err) - } - - return nil -} - -func (sb *sftpBlob) Write(p []byte) (int, error) { - n, err := sb.f.Write(p) - sb.size += uint(n) - return n, err -} - -func (sb *sftpBlob) Size() uint { - return sb.size -} - -// Create creates a new Blob. The data is available only after Finalize() -// has been called on the returned Blob. -func (r *SFTP) Create() (backend.Blob, error) { - // TODO: make sure that tempfile is removed upon error - - // create tempfile in backend - filename, file, err := r.tempFile() - if err != nil { - return nil, errors.Annotate(err, "create tempfile") - } - - blob := sftpBlob{ - f: file, - tempname: filename, - backend: r, - } - - return &blob, nil -} - +// Join joins the given paths and cleans them afterwards. func Join(parts ...string) string { return filepath.Clean(strings.Join(parts, "/")) } @@ -515,16 +459,16 @@ func (r *SFTP) List(t backend.Type, done <-chan struct{}) <-chan string { } // Close closes the sftp connection and terminates the underlying command. -func (s *SFTP) Close() error { - if s == nil { +func (r *SFTP) Close() error { + if r == nil { return nil } - s.c.Close() + r.c.Close() - if err := s.cmd.Process.Kill(); err != nil { + if err := r.cmd.Process.Kill(); err != nil { return err } - return s.cmd.Wait() + return r.cmd.Wait() } diff --git a/backend/test/backend_test.go b/backend/test/backend_test.go index e9c19680d..c1bee84c7 100644 --- a/backend/test/backend_test.go +++ b/backend/test/backend_test.go @@ -51,13 +51,6 @@ func TestTestBackendLoad(t *testing.T) { test.TestLoad(t) } -func TestTestBackendWrite(t *testing.T) { - if SkipMessage != "" { - t.Skip(SkipMessage) - } - test.TestWrite(t) -} - func TestTestBackendSave(t *testing.T) { if SkipMessage != "" { t.Skip(SkipMessage) diff --git a/backend/test/tests.go b/backend/test/tests.go index b3557d10d..f1b4b8107 100644 --- a/backend/test/tests.go +++ b/backend/test/tests.go @@ -156,19 +156,9 @@ func TestConfig(t testing.TB) { t.Fatalf("did not get expected error for non-existing config") } - blob, err := b.Create() + err = b.Save(backend.Handle{Type: backend.Config}, []byte(testString)) if err != nil { - t.Fatalf("Create() error: %v", err) - } - - _, err = blob.Write([]byte(testString)) - if err != nil { - t.Fatalf("Write() error: %v", err) - } - - err = blob.Finalize(backend.Config, "") - if err != nil { - t.Fatalf("Finalize() error: %v", err) + t.Fatalf("Save() error: %v", err) } // try accessing the config with different names, should all return the @@ -205,12 +195,11 @@ func TestLoad(t testing.TB) { data := Random(23, length) id := backend.Hash(data) - blob, err := b.Create() - OK(t, err) - - _, err = blob.Write([]byte(data)) - OK(t, err) - OK(t, blob.Finalize(backend.Data, id.String())) + handle := backend.Handle{Type: backend.Data, Name: id.String()} + err = b.Save(handle, data) + if err != nil { + t.Fatalf("Save() error: %v", err) + } for i := 0; i < 500; i++ { l := rand.Intn(length + 2000) @@ -229,8 +218,7 @@ func TestLoad(t testing.TB) { } buf := make([]byte, l) - h := backend.Handle{Type: backend.Data, Name: id.String()} - n, err := b.Load(h, buf, int64(o)) + n, err := b.Load(handle, buf, int64(o)) // if we requested data beyond the end of the file, ignore // ErrUnexpectedEOF error @@ -260,63 +248,6 @@ func TestLoad(t testing.TB) { OK(t, b.Remove(backend.Data, id.String())) } -// TestWrite tests writing data to the backend. -func TestWrite(t testing.TB) { - b := open(t) - defer close(t) - - length := rand.Intn(1<<23) + 2000 - - data := Random(23, length) - id := backend.Hash(data) - - for i := 0; i < 10; i++ { - blob, err := b.Create() - OK(t, err) - - o := 0 - for o < len(data) { - l := rand.Intn(len(data) - o) - if len(data)-o < 20 { - l = len(data) - o - } - - n, err := blob.Write(data[o : o+l]) - OK(t, err) - if n != l { - t.Fatalf("wrong number of bytes written, want %v, got %v", l, n) - } - - o += l - } - - name := fmt.Sprintf("%s-%d", id, i) - OK(t, blob.Finalize(backend.Data, name)) - - buf, err := backend.LoadAll(b, backend.Handle{Type: backend.Data, Name: name}, nil) - OK(t, err) - if len(buf) != len(data) { - t.Fatalf("number of bytes does not match, want %v, got %v", len(data), len(buf)) - } - - if !bytes.Equal(buf, data) { - t.Fatalf("data not equal") - } - - fi, err := b.Stat(backend.Handle{Type: backend.Data, Name: name}) - OK(t, err) - - if fi.Size != int64(len(data)) { - t.Fatalf("Stat() returned different size, want %q, got %d", len(data), fi.Size) - } - - err = b.Remove(backend.Data, name) - if err != nil { - t.Fatalf("error removing item: %v", err) - } - } -} - // TestSave tests saving data in the backend. func TestSave(t testing.TB) { b := open(t) @@ -415,13 +346,8 @@ var testStrings = []struct { func store(t testing.TB, b backend.Backend, tpe backend.Type, data []byte) { id := backend.Hash(data) - - blob, err := b.Create() + err := b.Save(backend.Handle{Name: id.String(), Type: tpe}, data) OK(t, err) - - _, err = blob.Write([]byte(data)) - OK(t, err) - OK(t, blob.Finalize(tpe, id.String())) } func read(t testing.TB, rd io.Reader, expectedData []byte) { @@ -492,12 +418,7 @@ func TestBackend(t testing.TB) { test := testStrings[0] // create blob - blob, err := b.Create() - OK(t, err) - - _, err = blob.Write([]byte(test.data)) - OK(t, err) - err = blob.Finalize(tpe, test.id) + err := b.Save(backend.Handle{Type: tpe, Name: test.id}, []byte(test.data)) Assert(t, err != nil, "expected error, got %v", err) // remove and recreate @@ -510,13 +431,9 @@ func TestBackend(t testing.TB) { Assert(t, ok == false, "removed blob still present") // create blob - blob, err = b.Create() + err = b.Save(backend.Handle{Type: tpe, Name: test.id}, []byte(test.data)) OK(t, err) - _, err = io.Copy(blob, bytes.NewReader([]byte(test.data))) - OK(t, err) - OK(t, blob.Finalize(tpe, test.id)) - // list items IDs := backend.IDs{} diff --git a/backend/utils_test.go b/backend/utils_test.go index 426f866de..98a0106ef 100644 --- a/backend/utils_test.go +++ b/backend/utils_test.go @@ -20,14 +20,9 @@ func TestLoadAll(t *testing.T) { data := Random(23+i, rand.Intn(MiB)+500*KiB) id := backend.Hash(data) - - blob, err := b.Create() + err := b.Save(backend.Handle{Name: id.String(), Type: backend.Data}, data) OK(t, err) - _, err = blob.Write([]byte(data)) - OK(t, err) - OK(t, blob.Finalize(backend.Data, id.String())) - buf, err := backend.LoadAll(b, backend.Handle{Type: backend.Data, Name: id.String()}, nil) OK(t, err) From d9c87559b528e44e350bb5e38dfdab3c9689ac35 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 21:13:24 +0100 Subject: [PATCH 39/56] s3/local backend: Fix error for overwriting files --- backend/local/local.go | 13 ++++++++----- backend/s3/s3.go | 8 ++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/backend/local/local.go b/backend/local/local.go index 9a6409a75..b98f118de 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -171,16 +171,19 @@ func (b *Local) Save(h backend.Handle, p []byte) (err error) { f := filename(b.p, h.Type, h.Name) - // create directories if necessary, ignore errors - if h.Type == backend.Data { - os.MkdirAll(filepath.Dir(f), backend.Modes.Dir) - } - // test if new path already exists if _, err := os.Stat(f); err == nil { return fmt.Errorf("Rename(): file %v already exists", f) } + // create directories if necessary, ignore errors + if h.Type == backend.Data { + err = os.MkdirAll(filepath.Dir(f), backend.Modes.Dir) + if err != nil { + return err + } + } + err = os.Rename(tmpfile.Name(), f) debug.Log("local.Save", "save %v: rename %v -> %v: %v", h, filepath.Base(tmpfile.Name()), filepath.Base(f), err) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index a8f69c4c9..097981b79 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -2,6 +2,7 @@ package s3 import ( "bytes" + "errors" "io" "strings" @@ -103,6 +104,13 @@ func (be S3Backend) Save(h backend.Handle, p []byte) (err error) { path := s3path(h.Type, h.Name) + // Check key does not already exist + _, err = be.client.StatObject(be.bucketname, path) + if err == nil { + debug.Log("s3.blob.Finalize()", "%v already exists", h) + return errors.New("key already exists") + } + <-be.connChan defer func() { be.connChan <- struct{}{} From a0d484113a08a474bf05f87a17c7af54f852d007 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 21:32:45 +0100 Subject: [PATCH 40/56] backends: Do not sort strings Closes #305 --- backend/local/local.go | 4 ---- backend/mem/mem_backend.go | 3 --- backend/sftp/sftp.go | 7 ------- 3 files changed, 14 deletions(-) diff --git a/backend/local/local.go b/backend/local/local.go index b98f118de..49dcb4ccd 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -7,7 +7,6 @@ import ( "io/ioutil" "os" "path/filepath" - "sort" "sync" "github.com/restic/restic/backend" @@ -253,7 +252,6 @@ func (b *Local) Remove(t backend.Type, name string) error { // goroutine is started for this. If the channel done is closed, sending // stops. func (b *Local) List(t backend.Type, done <-chan struct{}) <-chan string { - // TODO: use os.Open() and d.Readdirnames() instead of Glob() var pattern string if t == backend.Data { pattern = filepath.Join(dirname(b.p, t, ""), "*", "*") @@ -272,8 +270,6 @@ func (b *Local) List(t backend.Type, done <-chan struct{}) <-chan string { matches[i] = filepath.Base(matches[i]) } - sort.Strings(matches) - go func() { defer close(ch) for _, m := range matches { diff --git a/backend/mem/mem_backend.go b/backend/mem/mem_backend.go index 4de0afe82..4e8d3b46c 100644 --- a/backend/mem/mem_backend.go +++ b/backend/mem/mem_backend.go @@ -4,7 +4,6 @@ import ( "bytes" "errors" "io" - "sort" "sync" "github.com/restic/restic/backend" @@ -232,8 +231,6 @@ func memList(be *MemoryBackend, t backend.Type, done <-chan struct{}) <-chan str ids = append(ids, entry.Name) } - sort.Strings(ids) - debug.Log("MemoryBackend.List", "list %v: %v", t, ids) go func() { diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 94cc91132..8afc7b7b0 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -9,7 +9,6 @@ import ( "os" "os/exec" "path/filepath" - "sort" "strings" "github.com/juju/errors" @@ -407,8 +406,6 @@ func (r *SFTP) List(t backend.Type, done <-chan struct{}) <-chan string { dirs = append(dirs, d.Name()) } - sort.Strings(dirs) - // read files for _, dir := range dirs { entries, err := r.c.ReadDir(Join(basedir, dir)) @@ -421,8 +418,6 @@ func (r *SFTP) List(t backend.Type, done <-chan struct{}) <-chan string { items = append(items, entry.Name()) } - sort.Strings(items) - for _, file := range items { select { case ch <- file: @@ -442,8 +437,6 @@ func (r *SFTP) List(t backend.Type, done <-chan struct{}) <-chan string { items = append(items, entry.Name()) } - sort.Strings(items) - for _, file := range items { select { case ch <- file: From 5fcb5ae5492209cc01e300bc63c678b202453e7c Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 24 Jan 2016 21:40:54 +0100 Subject: [PATCH 41/56] Reduce number of tests for Load() --- backend/test/tests.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/test/tests.go b/backend/test/tests.go index f1b4b8107..e71412381 100644 --- a/backend/test/tests.go +++ b/backend/test/tests.go @@ -201,7 +201,7 @@ func TestLoad(t testing.TB) { t.Fatalf("Save() error: %v", err) } - for i := 0; i < 500; i++ { + for i := 0; i < 50; i++ { l := rand.Intn(length + 2000) o := rand.Intn(length + 2000) From b482df04ec070affb66cc63aa6c1ccbc2465c05b Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 26 Jan 2016 21:49:22 +0100 Subject: [PATCH 42/56] Add more documentation --- backend/doc.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/doc.go b/backend/doc.go index e85cef355..f82c3d671 100644 --- a/backend/doc.go +++ b/backend/doc.go @@ -1,2 +1,5 @@ // Package backend provides local and remote storage for restic repositories. +// All backends need to implement the Backend interface. There is a +// MockBackend, which can be used for mocking in tests, and a MemBackend, which +// stores all data in a hash internally. package backend From da883d61965def4ee871ecdbb2170c8a86ed8ac3 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 26 Jan 2016 21:49:33 +0100 Subject: [PATCH 43/56] Cleanups, move Hash() to id.go --- backend/generic.go | 33 ++++++++++----------------------- backend/id.go | 12 +++++++----- 2 files changed, 17 insertions(+), 28 deletions(-) diff --git a/backend/generic.go b/backend/generic.go index d8c72b9e0..c528f8998 100644 --- a/backend/generic.go +++ b/backend/generic.go @@ -1,29 +1,14 @@ package backend -import ( - "crypto/sha256" - "errors" -) +import "errors" -const ( - MinPrefixLength = 8 -) +// ErrNoIDPrefixFound is returned by Find() when no ID for the given prefix +// could be found. +var ErrNoIDPrefixFound = errors.New("no ID found") -var ( - ErrNoIDPrefixFound = errors.New("no ID found") - ErrMultipleIDMatches = errors.New("multiple IDs with prefix found") -) - -var ( - hashData = sha256.Sum256 -) - -const hashSize = sha256.Size - -// Hash returns the ID for data. -func Hash(data []byte) ID { - return hashData(data) -} +// ErrMultipleIDMatches is returned by Find() when multiple IDs with the given +// prefix are found. +var ErrMultipleIDMatches = errors.New("multiple IDs with prefix found") // Find loads the list of all blobs of type t and searches for names which // start with prefix. If none is found, nil and ErrNoIDPrefixFound is returned. @@ -52,6 +37,8 @@ func Find(be Lister, t Type, prefix string) (string, error) { return "", ErrNoIDPrefixFound } +const minPrefixLength = 8 + // PrefixLength returns the number of bytes required so that all prefixes of // all names of type t are unique. func PrefixLength(be Lister, t Type) (int, error) { @@ -66,7 +53,7 @@ func PrefixLength(be Lister, t Type) (int, error) { // select prefixes of length l, test if the last one is the same as the current one outer: - for l := MinPrefixLength; l < IDSize; l++ { + for l := minPrefixLength; l < IDSize; l++ { var last string for _, name := range list { diff --git a/backend/id.go b/backend/id.go index 966cd7a4e..e76d264f5 100644 --- a/backend/id.go +++ b/backend/id.go @@ -2,13 +2,19 @@ package backend import ( "bytes" + "crypto/sha256" "encoding/hex" "encoding/json" "errors" ) +// Hash returns the ID for data. +func Hash(data []byte) ID { + return sha256.Sum256(data) +} + // IDSize contains the size of an ID, in bytes. -const IDSize = hashSize +const IDSize = sha256.Size // ID references content within a repository. type ID [IDSize]byte @@ -98,7 +104,3 @@ func (id *ID) UnmarshalJSON(b []byte) error { return nil } - -func IDFromData(d []byte) ID { - return hashData(d) -} From c34aa7253862242fede25eb6c88c676911449e72 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 26 Jan 2016 21:52:02 +0100 Subject: [PATCH 44/56] Remove duplicate function str2id --- backend/generic_test.go | 25 ++++++++----------------- backend/id.go | 2 ++ backend/ids_test.go | 35 ++++++++++++++++++----------------- backend/idset_test.go | 21 +++++++++++---------- 4 files changed, 39 insertions(+), 44 deletions(-) diff --git a/backend/generic_test.go b/backend/generic_test.go index cd401516c..ca5b78982 100644 --- a/backend/generic_test.go +++ b/backend/generic_test.go @@ -7,15 +7,6 @@ import ( . "github.com/restic/restic/test" ) -func str2id(s string) backend.ID { - id, err := backend.ParseID(s) - if err != nil { - panic(err) - } - - return id -} - type mockBackend struct { list func(backend.Type, <-chan struct{}) <-chan string } @@ -25,14 +16,14 @@ func (m mockBackend) List(t backend.Type, done <-chan struct{}) <-chan string { } var samples = backend.IDs{ - str2id("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff"), - str2id("20bdc1402a6fc9b633ccd578c4a92d0f4ef1a457fa2e16c596bc73fb409d6cc0"), - str2id("20bdc1402a6fc9b633ffffffffffffffffffffffffffffffffffffffffffffff"), - str2id("20ff988befa5fc40350f00d531a767606efefe242c837aaccb80673f286be53d"), - str2id("326cb59dfe802304f96ee9b5b9af93bdee73a30f53981e5ec579aedb6f1d0f07"), - str2id("86b60b9594d1d429c4aa98fa9562082cabf53b98c7dc083abe5dae31074dd15a"), - str2id("96c8dbe225079e624b5ce509f5bd817d1453cd0a85d30d536d01b64a8669aeae"), - str2id("fa31d65b87affcd167b119e9d3d2a27b8236ca4836cb077ed3e96fcbe209b792"), + ParseID("20bdc1402a6fc9b633aaffffffffffffffffffffffffffffffffffffffffffff"), + ParseID("20bdc1402a6fc9b633ccd578c4a92d0f4ef1a457fa2e16c596bc73fb409d6cc0"), + ParseID("20bdc1402a6fc9b633ffffffffffffffffffffffffffffffffffffffffffffff"), + ParseID("20ff988befa5fc40350f00d531a767606efefe242c837aaccb80673f286be53d"), + ParseID("326cb59dfe802304f96ee9b5b9af93bdee73a30f53981e5ec579aedb6f1d0f07"), + ParseID("86b60b9594d1d429c4aa98fa9562082cabf53b98c7dc083abe5dae31074dd15a"), + ParseID("96c8dbe225079e624b5ce509f5bd817d1453cd0a85d30d536d01b64a8669aeae"), + ParseID("fa31d65b87affcd167b119e9d3d2a27b8236ca4836cb077ed3e96fcbe209b792"), } func TestPrefixLength(t *testing.T) { diff --git a/backend/id.go b/backend/id.go index e76d264f5..115792707 100644 --- a/backend/id.go +++ b/backend/id.go @@ -86,10 +86,12 @@ func (id ID) Compare(other ID) int { return bytes.Compare(other[:], id[:]) } +// MarshalJSON returns the JSON encoding of id. func (id ID) MarshalJSON() ([]byte, error) { return json.Marshal(id.String()) } +// UnmarshalJSON parses the JSON-encoded data and stores the result in id. func (id *ID) UnmarshalJSON(b []byte) error { var s string err := json.Unmarshal(b, &s) diff --git a/backend/ids_test.go b/backend/ids_test.go index 02647eba1..eac56d30c 100644 --- a/backend/ids_test.go +++ b/backend/ids_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/restic/restic/backend" + . "github.com/restic/restic/test" ) var uniqTests = []struct { @@ -12,37 +13,37 @@ var uniqTests = []struct { }{ { backend.IDs{ - str2id("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), - str2id("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), - str2id("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), + ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), + ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), + ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), }, backend.IDs{ - str2id("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), - str2id("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), + ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), + ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), }, }, { backend.IDs{ - str2id("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), - str2id("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), - str2id("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), + ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), + ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), + ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), }, backend.IDs{ - str2id("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), - str2id("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), + ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), + ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), }, }, { backend.IDs{ - str2id("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), - str2id("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"), - str2id("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), - str2id("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), + ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), + ParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"), + ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), + ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), }, backend.IDs{ - str2id("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), - str2id("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"), - str2id("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), + ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), + ParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"), + ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), }, }, } diff --git a/backend/idset_test.go b/backend/idset_test.go index 7084c8abf..6659c2229 100644 --- a/backend/idset_test.go +++ b/backend/idset_test.go @@ -4,22 +4,23 @@ import ( "testing" "github.com/restic/restic/backend" + . "github.com/restic/restic/test" ) var idsetTests = []struct { id backend.ID seen bool }{ - {str2id("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), false}, - {str2id("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), false}, - {str2id("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true}, - {str2id("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true}, - {str2id("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), true}, - {str2id("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"), false}, - {str2id("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true}, - {str2id("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), true}, - {str2id("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"), true}, - {str2id("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true}, + {ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), false}, + {ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), false}, + {ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true}, + {ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true}, + {ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), true}, + {ParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"), false}, + {ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true}, + {ParseID("1285b30394f3b74693cc29a758d9624996ae643157776fce8154aabd2f01515f"), true}, + {ParseID("f658198b405d7e80db5ace1980d125c8da62f636b586c46bf81dfb856a49d0c8"), true}, + {ParseID("7bb086db0d06285d831485da8031281e28336a56baa313539eaea1c73a2a1a40"), true}, } func TestIDSet(t *testing.T) { From eb1669a061a0484027fde9b2ce6902b36935a43f Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 26 Jan 2016 21:56:13 +0100 Subject: [PATCH 45/56] Add a lot of comments --- backend/mock_backend.go | 9 +++++++++ backend/paths.go | 5 +++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/backend/mock_backend.go b/backend/mock_backend.go index e685ba556..921af4d6c 100644 --- a/backend/mock_backend.go +++ b/backend/mock_backend.go @@ -16,6 +16,7 @@ type MockBackend struct { LocationFn func() string } +// Close the backend. func (m *MockBackend) Close() error { if m.CloseFn == nil { return nil @@ -24,6 +25,7 @@ func (m *MockBackend) Close() error { return m.CloseFn() } +// Location returns a location string. func (m *MockBackend) Location() string { if m.LocationFn == nil { return "" @@ -32,6 +34,7 @@ func (m *MockBackend) Location() string { return m.LocationFn() } +// Load loads data from the backend. func (m *MockBackend) Load(h Handle, p []byte, off int64) (int, error) { if m.LoadFn == nil { return 0, errors.New("not implemented") @@ -40,6 +43,7 @@ func (m *MockBackend) Load(h Handle, p []byte, off int64) (int, error) { return m.LoadFn(h, p, off) } +// Save data in the backend. func (m *MockBackend) Save(h Handle, p []byte) error { if m.SaveFn == nil { return errors.New("not implemented") @@ -48,6 +52,7 @@ func (m *MockBackend) Save(h Handle, p []byte) error { return m.SaveFn(h, p) } +// Stat an object in the backend. func (m *MockBackend) Stat(h Handle) (BlobInfo, error) { if m.StatFn == nil { return BlobInfo{}, errors.New("not implemented") @@ -56,6 +61,7 @@ func (m *MockBackend) Stat(h Handle) (BlobInfo, error) { return m.StatFn(h) } +// List items of type t. func (m *MockBackend) List(t Type, done <-chan struct{}) <-chan string { if m.ListFn == nil { ch := make(chan string) @@ -66,6 +72,7 @@ func (m *MockBackend) List(t Type, done <-chan struct{}) <-chan string { return m.ListFn(t, done) } +// Remove data from the backend. func (m *MockBackend) Remove(t Type, name string) error { if m.RemoveFn == nil { return errors.New("not implemented") @@ -74,6 +81,7 @@ func (m *MockBackend) Remove(t Type, name string) error { return m.RemoveFn(t, name) } +// Test for the existence of a specific item. func (m *MockBackend) Test(t Type, name string) (bool, error) { if m.TestFn == nil { return false, errors.New("not implemented") @@ -82,6 +90,7 @@ func (m *MockBackend) Test(t Type, name string) (bool, error) { return m.TestFn(t, name) } +// Delete all data. func (m *MockBackend) Delete() error { if m.DeleteFn == nil { return errors.New("not implemented") diff --git a/backend/paths.go b/backend/paths.go index 8e29e6950..940e9fcb9 100644 --- a/backend/paths.go +++ b/backend/paths.go @@ -2,7 +2,7 @@ package backend import "os" -// Default paths for file-based backends (e.g. local) +// Paths contains the default paths for file-based backends (e.g. local). var Paths = struct { Data string Snapshots string @@ -21,5 +21,6 @@ var Paths = struct { "config", } -// Default modes for file-based backends +// Modes holds the default modes for directories and files for file-based +// backends. var Modes = struct{ Dir, File os.FileMode }{0700, 0600} From 7196971159eec0585cd3ca8da683d6be0a4f4783 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 26 Jan 2016 22:00:11 +0100 Subject: [PATCH 46/56] Remove unneeded HashingReader implementation --- backend/writer.go | 38 ----------------------------------- backend/writer_test.go | 45 ------------------------------------------ 2 files changed, 83 deletions(-) delete mode 100644 backend/writer.go delete mode 100644 backend/writer_test.go diff --git a/backend/writer.go b/backend/writer.go deleted file mode 100644 index 5764b872c..000000000 --- a/backend/writer.go +++ /dev/null @@ -1,38 +0,0 @@ -package backend - -import ( - "hash" - "io" -) - -// HashingWriter wraps an io.Writer to hashes all data that is written to it. -type HashingWriter struct { - w io.Writer - h hash.Hash - size int -} - -// NewHashAppendWriter wraps the writer w and feeds all data written to the hash h. -func NewHashingWriter(w io.Writer, h hash.Hash) *HashingWriter { - return &HashingWriter{ - h: h, - w: io.MultiWriter(w, h), - } -} - -// Write wraps the write method of the underlying writer and also hashes all data. -func (h *HashingWriter) Write(p []byte) (int, error) { - n, err := h.w.Write(p) - h.size += n - return n, err -} - -// Sum returns the hash of all data written so far. -func (h *HashingWriter) Sum(d []byte) []byte { - return h.h.Sum(d) -} - -// Size returns the number of bytes written to the underlying writer. -func (h *HashingWriter) Size() int { - return h.size -} diff --git a/backend/writer_test.go b/backend/writer_test.go deleted file mode 100644 index 9fda2c06f..000000000 --- a/backend/writer_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package backend_test - -import ( - "bytes" - "crypto/rand" - "crypto/sha256" - "io" - "io/ioutil" - "testing" - - "github.com/restic/restic/backend" - . "github.com/restic/restic/test" -) - -func TestHashingWriter(t *testing.T) { - tests := []int{5, 23, 2<<18 + 23, 1 << 20} - - for _, size := range tests { - data := make([]byte, size) - _, err := io.ReadFull(rand.Reader, data) - if err != nil { - t.Fatalf("ReadFull: %v", err) - } - - expectedHash := sha256.Sum256(data) - - wr := backend.NewHashingWriter(ioutil.Discard, sha256.New()) - - n, err := io.Copy(wr, bytes.NewReader(data)) - OK(t, err) - - Assert(t, n == int64(size), - "HashAppendWriter: invalid number of bytes written: got %d, expected %d", - n, size) - - Assert(t, wr.Size() == size, - "HashAppendWriter: invalid number of bytes returned: got %d, expected %d", - wr.Size, size) - - resultingHash := wr.Sum(nil) - Assert(t, bytes.Equal(expectedHash[:], resultingHash), - "HashAppendWriter: hashes do not match: expected %02x, got %02x", - expectedHash, resultingHash) - } -} From 9b1c4b2dd6076136c9b017ec32157be9324c859d Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 26 Jan 2016 22:07:51 +0100 Subject: [PATCH 47/56] local: Remove mutex and hash of open files --- backend/local/local.go | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/backend/local/local.go b/backend/local/local.go index 49dcb4ccd..80ea65e2f 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -7,7 +7,6 @@ import ( "io/ioutil" "os" "path/filepath" - "sync" "github.com/restic/restic/backend" "github.com/restic/restic/debug" @@ -15,9 +14,7 @@ import ( // Local is a backend in a local directory. type Local struct { - p string - mu sync.Mutex - open map[string][]*os.File // Contains open files. Guarded by 'mu'. + p string } // Open opens the local backend as specified by config. @@ -39,7 +36,7 @@ func Open(dir string) (*Local, error) { } } - return &Local{p: dir, open: make(map[string][]*os.File)}, nil + return &Local{p: dir}, nil } // Create creates all the necessary files and directories for a new local @@ -229,15 +226,7 @@ func (b *Local) Test(t backend.Type, name string) (bool, error) { // Remove removes the blob with the given name and type. func (b *Local) Remove(t backend.Type, name string) error { - // close all open files we may have. fn := filename(b.p, t, name) - b.mu.Lock() - open, _ := b.open[fn] - for _, file := range open { - file.Close() - } - b.open[fn] = nil - b.mu.Unlock() // reset read-only flag err := os.Chmod(fn, 0666) @@ -290,21 +279,12 @@ func (b *Local) List(t backend.Type, done <-chan struct{}) <-chan string { // Delete removes the repository and all files. func (b *Local) Delete() error { - b.Close() return os.RemoveAll(b.p) } // Close closes all open files. -// They may have been closed already, -// so we ignore all errors. func (b *Local) Close() error { - b.mu.Lock() - for _, open := range b.open { - for _, file := range open { - file.Close() - } - } - b.open = make(map[string][]*os.File) - b.mu.Unlock() + // this does not need to do anything, all open files are closed within the + // same function. return nil } From 9ec435d86361dc6fa51f299ef6f36642159bf06a Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 26 Jan 2016 22:09:29 +0100 Subject: [PATCH 48/56] local: remove duplicate code --- backend/local/local.go | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/backend/local/local.go b/backend/local/local.go index 80ea65e2f..6040bdfdc 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -17,9 +17,8 @@ type Local struct { p string } -// Open opens the local backend as specified by config. -func Open(dir string) (*Local, error) { - items := []string{ +func paths(dir string) []string { + return []string{ dir, filepath.Join(dir, backend.Paths.Data), filepath.Join(dir, backend.Paths.Snapshots), @@ -28,9 +27,12 @@ func Open(dir string) (*Local, error) { filepath.Join(dir, backend.Paths.Keys), filepath.Join(dir, backend.Paths.Temp), } +} +// Open opens the local backend as specified by config. +func Open(dir string) (*Local, error) { // test if all necessary dirs are there - for _, d := range items { + for _, d := range paths(dir) { if _, err := os.Stat(d); err != nil { return nil, fmt.Errorf("%s does not exist", d) } @@ -42,16 +44,6 @@ func Open(dir string) (*Local, error) { // Create creates all the necessary files and directories for a new local // backend at dir. Afterwards a new config blob should be created. func Create(dir string) (*Local, error) { - dirs := []string{ - dir, - filepath.Join(dir, backend.Paths.Data), - filepath.Join(dir, backend.Paths.Snapshots), - filepath.Join(dir, backend.Paths.Index), - filepath.Join(dir, backend.Paths.Locks), - filepath.Join(dir, backend.Paths.Keys), - filepath.Join(dir, backend.Paths.Temp), - } - // test if config file already exists _, err := os.Lstat(filepath.Join(dir, backend.Paths.Config)) if err == nil { @@ -59,7 +51,7 @@ func Create(dir string) (*Local, error) { } // create paths for data, refs and temp - for _, d := range dirs { + for _, d := range paths(dir) { err := os.MkdirAll(d, backend.Modes.Dir) if err != nil { return nil, err From 0bbad683c548834a2050d210f8a42f6a614e4e48 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 26 Jan 2016 22:12:53 +0100 Subject: [PATCH 49/56] local: split out tempfile write function --- backend/local/local.go | 70 +++++++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/backend/local/local.go b/backend/local/local.go index 6040bdfdc..521ffde79 100644 --- a/backend/local/local.go +++ b/backend/local/local.go @@ -126,67 +126,73 @@ func (b *Local) Load(h backend.Handle, p []byte, off int64) (n int, err error) { return io.ReadFull(f, p) } +// writeToTempfile saves p into a tempfile in tempdir. +func writeToTempfile(tempdir string, p []byte) (filename string, err error) { + tmpfile, err := ioutil.TempFile(tempdir, "temp-") + if err != nil { + return "", err + } + + n, err := tmpfile.Write(p) + if err != nil { + return "", err + } + + if n != len(p) { + return "", errors.New("not all bytes writen") + } + + if err = tmpfile.Sync(); err != nil { + return "", err + } + + err = tmpfile.Close() + if err != nil { + return "", err + } + + return tmpfile.Name(), nil +} + // Save stores data in the backend at the handle. func (b *Local) Save(h backend.Handle, p []byte) (err error) { if err := h.Valid(); err != nil { return err } - tmpfile, err := ioutil.TempFile(filepath.Join(b.p, backend.Paths.Temp), "temp-") - if err != nil { - return err - } + tmpfile, err := writeToTempfile(filepath.Join(b.p, backend.Paths.Temp), p) + debug.Log("local.Save", "saved %v (%d bytes) to %v", h, len(p), tmpfile) - debug.Log("local.Save", "save %v (%d bytes) to %v", h, len(p), tmpfile.Name()) - - n, err := tmpfile.Write(p) - if err != nil { - return err - } - - if n != len(p) { - return errors.New("not all bytes writen") - } - - if err = tmpfile.Sync(); err != nil { - return err - } - - err = tmpfile.Close() - if err != nil { - return err - } - - f := filename(b.p, h.Type, h.Name) + filename := filename(b.p, h.Type, h.Name) // test if new path already exists - if _, err := os.Stat(f); err == nil { - return fmt.Errorf("Rename(): file %v already exists", f) + if _, err := os.Stat(filename); err == nil { + return fmt.Errorf("Rename(): file %v already exists", filename) } // create directories if necessary, ignore errors if h.Type == backend.Data { - err = os.MkdirAll(filepath.Dir(f), backend.Modes.Dir) + err = os.MkdirAll(filepath.Dir(filename), backend.Modes.Dir) if err != nil { return err } } - err = os.Rename(tmpfile.Name(), f) + err = os.Rename(tmpfile, filename) debug.Log("local.Save", "save %v: rename %v -> %v: %v", - h, filepath.Base(tmpfile.Name()), filepath.Base(f), err) + h, filepath.Base(tmpfile), filepath.Base(filename), err) if err != nil { return err } // set mode to read-only - fi, err := os.Stat(f) + fi, err := os.Stat(filename) if err != nil { return err } - return setNewFileMode(f, fi) + return setNewFileMode(filename, fi) } // Stat returns information about a blob. From 1528d1ca8343da71ed4665ce55373ed421b52b0a Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 26 Jan 2016 22:16:17 +0100 Subject: [PATCH 50/56] sftp: Reduce duplicate code, add error check --- backend/sftp/sftp.go | 38 +++++++++++++------------------ backend/sftp/sftp_backend_test.go | 3 +-- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/backend/sftp/sftp.go b/backend/sftp/sftp.go index 8afc7b7b0..fb10c3854 100644 --- a/backend/sftp/sftp.go +++ b/backend/sftp/sftp.go @@ -64,6 +64,18 @@ func startClient(program string, args ...string) (*SFTP, error) { return &SFTP{c: client, cmd: cmd}, nil } +func paths(dir string) []string { + return []string{ + dir, + Join(dir, backend.Paths.Data), + Join(dir, backend.Paths.Snapshots), + Join(dir, backend.Paths.Index), + Join(dir, backend.Paths.Locks), + Join(dir, backend.Paths.Keys), + Join(dir, backend.Paths.Temp), + } +} + // Open opens an sftp backend. When the command is started via // exec.Command, it is expected to speak sftp on stdin/stdout. The backend // is expected at the given path. @@ -74,16 +86,7 @@ func Open(dir string, program string, args ...string) (*SFTP, error) { } // test if all necessary dirs and files are there - items := []string{ - dir, - Join(dir, backend.Paths.Data), - Join(dir, backend.Paths.Snapshots), - Join(dir, backend.Paths.Index), - Join(dir, backend.Paths.Locks), - Join(dir, backend.Paths.Keys), - Join(dir, backend.Paths.Temp), - } - for _, d := range items { + for _, d := range paths(dir) { if _, err := sftp.c.Lstat(d); err != nil { return nil, fmt.Errorf("%s does not exist", d) } @@ -118,16 +121,6 @@ func Create(dir string, program string, args ...string) (*SFTP, error) { return nil, err } - dirs := []string{ - dir, - Join(dir, backend.Paths.Data), - Join(dir, backend.Paths.Snapshots), - Join(dir, backend.Paths.Index), - Join(dir, backend.Paths.Locks), - Join(dir, backend.Paths.Keys), - Join(dir, backend.Paths.Temp), - } - // test if config file already exists _, err = sftp.c.Lstat(Join(dir, backend.Paths.Config)) if err == nil { @@ -135,7 +128,7 @@ func Create(dir string, program string, args ...string) (*SFTP, error) { } // create paths for data, refs and temp blobs - for _, d := range dirs { + for _, d := range paths(dir) { err = sftp.mkdirAll(d, backend.Modes.Dir) if err != nil { return nil, err @@ -457,7 +450,8 @@ func (r *SFTP) Close() error { return nil } - r.c.Close() + err := r.c.Close() + debug.Log("sftp.Close", "Close returned error %v", err) if err := r.cmd.Process.Kill(); err != nil { return err diff --git a/backend/sftp/sftp_backend_test.go b/backend/sftp/sftp_backend_test.go index 431abff2a..bfb8e4e75 100644 --- a/backend/sftp/sftp_backend_test.go +++ b/backend/sftp/sftp_backend_test.go @@ -38,8 +38,7 @@ func init() { for _, dir := range strings.Split(TestSFTPPath, ":") { testpath := filepath.Join(dir, "sftp-server") - fd, err := os.Open(testpath) - fd.Close() + _, err := os.Stat(testpath) if !os.IsNotExist(err) { sftpserver = testpath break From c388101217639f5766dfacd4ba9306a7f68a3dbe Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 26 Jan 2016 22:19:10 +0100 Subject: [PATCH 51/56] s3: Unexport structure --- backend/s3/s3.go | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/backend/s3/s3.go b/backend/s3/s3.go index 097981b79..ccb6cd42c 100644 --- a/backend/s3/s3.go +++ b/backend/s3/s3.go @@ -12,7 +12,6 @@ import ( "github.com/restic/restic/debug" ) -const maxKeysInList = 1000 const connLimit = 10 const backendPrefix = "restic" @@ -23,8 +22,8 @@ func s3path(t backend.Type, name string) string { return backendPrefix + "/" + string(t) + "/" + name } -// S3Backend is a backend which stores the data on an S3 endpoint. -type S3Backend struct { +// s3 is a backend which stores the data on an S3 endpoint. +type s3 struct { client minio.CloudStorageClient connChan chan struct{} bucketname string @@ -40,7 +39,7 @@ func Open(cfg Config) (backend.Backend, error) { return nil, err } - be := &S3Backend{client: client, bucketname: cfg.Bucket} + be := &s3{client: client, bucketname: cfg.Bucket} be.createConnections() if err := client.BucketExists(cfg.Bucket); err != nil { @@ -57,7 +56,7 @@ func Open(cfg Config) (backend.Backend, error) { return be, nil } -func (be *S3Backend) createConnections() { +func (be *s3) createConnections() { be.connChan = make(chan struct{}, connLimit) for i := 0; i < connLimit; i++ { be.connChan <- struct{}{} @@ -65,13 +64,13 @@ func (be *S3Backend) createConnections() { } // Location returns this backend's location (the bucket name). -func (be *S3Backend) Location() string { +func (be *s3) Location() string { return be.bucketname } // Load returns the data stored in the backend for h at the given offset // and saves it in p. Load has the same semantics as io.ReaderAt. -func (be S3Backend) Load(h backend.Handle, p []byte, off int64) (int, error) { +func (be s3) Load(h backend.Handle, p []byte, off int64) (int, error) { debug.Log("s3.Load", "%v, offset %v, len %v", h, off, len(p)) path := s3path(h.Type, h.Name) obj, err := be.client.GetObject(be.bucketname, path) @@ -95,7 +94,7 @@ func (be S3Backend) Load(h backend.Handle, p []byte, off int64) (int, error) { } // Save stores data in the backend at the handle. -func (be S3Backend) Save(h backend.Handle, p []byte) (err error) { +func (be s3) Save(h backend.Handle, p []byte) (err error) { if err := h.Valid(); err != nil { return err } @@ -125,7 +124,7 @@ func (be S3Backend) Save(h backend.Handle, p []byte) (err error) { } // Stat returns information about a blob. -func (be S3Backend) Stat(h backend.Handle) (backend.BlobInfo, error) { +func (be s3) Stat(h backend.Handle) (backend.BlobInfo, error) { debug.Log("s3.Stat", "%v") path := s3path(h.Type, h.Name) obj, err := be.client.GetObject(be.bucketname, path) @@ -144,7 +143,7 @@ func (be S3Backend) Stat(h backend.Handle) (backend.BlobInfo, error) { } // Test returns true if a blob of the given type and name exists in the backend. -func (be *S3Backend) Test(t backend.Type, name string) (bool, error) { +func (be *s3) Test(t backend.Type, name string) (bool, error) { found := false path := s3path(t, name) _, err := be.client.StatObject(be.bucketname, path) @@ -157,7 +156,7 @@ func (be *S3Backend) Test(t backend.Type, name string) (bool, error) { } // Remove removes the blob with the given name and type. -func (be *S3Backend) Remove(t backend.Type, name string) error { +func (be *s3) Remove(t backend.Type, name string) error { path := s3path(t, name) err := be.client.RemoveObject(be.bucketname, path) debug.Log("s3.Remove", "%v %v -> err %v", t, name, err) @@ -167,7 +166,7 @@ func (be *S3Backend) Remove(t backend.Type, name string) error { // List returns a channel that yields all names of blobs of type t. A // goroutine is started for this. If the channel done is closed, sending // stops. -func (be *S3Backend) List(t backend.Type, done <-chan struct{}) <-chan string { +func (be *s3) List(t backend.Type, done <-chan struct{}) <-chan string { debug.Log("s3.List", "listing %v", t) ch := make(chan string) @@ -195,7 +194,7 @@ func (be *S3Backend) List(t backend.Type, done <-chan struct{}) <-chan string { } // Remove keys for a specified backend type. -func (be *S3Backend) removeKeys(t backend.Type) error { +func (be *s3) removeKeys(t backend.Type) error { done := make(chan struct{}) defer close(done) for key := range be.List(backend.Data, done) { @@ -209,7 +208,7 @@ func (be *S3Backend) removeKeys(t backend.Type) error { } // Delete removes all restic keys in the bucket. It will not remove the bucket itself. -func (be *S3Backend) Delete() error { +func (be *s3) Delete() error { alltypes := []backend.Type{ backend.Data, backend.Key, @@ -228,4 +227,4 @@ func (be *S3Backend) Delete() error { } // Close does nothing -func (be *S3Backend) Close() error { return nil } +func (be *s3) Close() error { return nil } From 2701eabe394cbea77e25d160facfb8c1ad3c05df Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 26 Jan 2016 22:35:51 +0100 Subject: [PATCH 52/56] Remove ContinuousReader --- backend/s3/cont_reader.go | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 backend/s3/cont_reader.go diff --git a/backend/s3/cont_reader.go b/backend/s3/cont_reader.go deleted file mode 100644 index 77bd1dca1..000000000 --- a/backend/s3/cont_reader.go +++ /dev/null @@ -1,16 +0,0 @@ -package s3 - -import "io" - -// ContinuousReader implements an io.Reader on top of an io.ReaderAt, advancing -// an offset. -type ContinuousReader struct { - R io.ReaderAt - Offset int64 -} - -func (c *ContinuousReader) Read(p []byte) (int, error) { - n, err := c.R.ReadAt(p, c.Offset) - c.Offset += int64(n) - return n, err -} From 1fde8720163c650ffa135467b0dfe9916d806ad5 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 26 Jan 2016 22:36:06 +0100 Subject: [PATCH 53/56] CI: only build minio on Go 1.5.1 and above --- run_integration_tests.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/run_integration_tests.go b/run_integration_tests.go index 5ea7957bc..27c1ee4c2 100644 --- a/run_integration_tests.go +++ b/run_integration_tests.go @@ -42,7 +42,9 @@ func (env *TravisEnvironment) Prepare() { run("go", "get", "golang.org/x/tools/cmd/cover") run("go", "get", "github.com/mattn/goveralls") run("go", "get", "github.com/pierrre/gotestcover") - runWithEnv(envVendorExperiment, "go", "get", "github.com/minio/minio") + if goVersionAtLeast151() { + runWithEnv(envVendorExperiment, "go", "get", "github.com/minio/minio") + } if runtime.GOOS == "darwin" { // install the libraries necessary for fuse From b64006221c7536edcfbb9477f5ec9900b8f23dac Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 26 Jan 2016 23:42:45 +0100 Subject: [PATCH 54/56] CI: Download minio server, do not compile latest master --- Dockerfile | 7 +++-- run_integration_tests.go | 63 ++++++++++++++++++++++++++++++++++------ 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index 03d8c9628..7b5a94cd4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,12 +41,15 @@ ENV PATH $PATH:$GOPATH/bin RUN mkdir -p $GOPATH/src/github.com/restic/restic -# install tools +# pre-install tools, this speeds up running the tests itself +RUN go get github.com/tools/godep RUN go get golang.org/x/tools/cmd/cover RUN go get github.com/mattn/goveralls RUN go get github.com/mitchellh/gox RUN go get github.com/pierrre/gotestcover -RUN GO15VENDOREXPERIMENT=1 go get github.com/minio/minio +RUN mkdir $HOME/bin \ + && wget -q -O $HOME/bin/minio https://dl.minio.io/server/minio/release/linux-amd64/minio \ + && chmod +x $HOME/bin/minio # set TRAVIS_BUILD_DIR for integration script ENV TRAVIS_BUILD_DIR $GOPATH/src/github.com/restic/restic diff --git a/run_integration_tests.go b/run_integration_tests.go index 27c1ee4c2..0a4c59401 100644 --- a/run_integration_tests.go +++ b/run_integration_tests.go @@ -6,7 +6,9 @@ import ( "bytes" "flag" "fmt" + "io" "io/ioutil" + "net/http" "os" "os/exec" "path/filepath" @@ -17,6 +19,7 @@ import ( ) var runCrossCompile = flag.Bool("cross-compile", true, "run cross compilation tests") +var minioServer = flag.String("minio", "", "path to the minio server binary") func init() { flag.Parse() @@ -30,10 +33,54 @@ type CIEnvironment interface { type TravisEnvironment struct { goxArch []string goxOS []string + minio string } -var envVendorExperiment = map[string]string{ - "GO15VENDOREXPERIMENT": "1", +func (env *TravisEnvironment) getMinio() { + if *minioServer != "" { + msg("using minio server at %q\n", *minioServer) + env.minio = *minioServer + return + } + + tempfile, err := ioutil.TempFile("", "minio-server-") + if err != nil { + fmt.Fprintf(os.Stderr, "create tempfile failed: %v\n", err) + os.Exit(10) + } + + res, err := http.Get("https://dl.minio.io/server/minio/release/linux-amd64/minio") + if err != nil { + msg("downloading minio failed: %v\n", err) + return + } + + _, err = io.Copy(tempfile, res.Body) + if err != nil { + msg("downloading minio failed: %v\n", err) + return + } + + err = res.Body.Close() + if err != nil { + msg("saving minio failed: %v\n", err) + return + } + + err = tempfile.Close() + if err != nil { + msg("closing tempfile failed: %v\n", err) + return + } + + err = os.Chmod(tempfile.Name(), 0755) + if err != nil { + msg("making minio server executable failed: %v\n", err) + return + } + + msg("downloaded minio server to %v\n", tempfile.Name()) + env.minio = tempfile.Name() } func (env *TravisEnvironment) Prepare() { @@ -42,9 +89,7 @@ func (env *TravisEnvironment) Prepare() { run("go", "get", "golang.org/x/tools/cmd/cover") run("go", "get", "github.com/mattn/goveralls") run("go", "get", "github.com/pierrre/gotestcover") - if goVersionAtLeast151() { - runWithEnv(envVendorExperiment, "go", "get", "github.com/minio/minio") - } + env.getMinio() if runtime.GOOS == "darwin" { // install the libraries necessary for fuse @@ -127,8 +172,8 @@ func (env *TravisEnvironment) RunTests() { err error ) - if goVersionAtLeast151() { - srv, err = NewMinioServer() + if env.minio != "" { + srv, err = NewMinioServer(env.minio) if err != nil { fmt.Fprintf(os.Stderr, "error running minio server: %v", err) os.Exit(8) @@ -275,7 +320,7 @@ var minioEnv = map[string]string{ // NewMinioServer prepares and runs a minio server for the s3 backend tests in // a temporary directory. -func NewMinioServer() (*MinioServer, error) { +func NewMinioServer(minio string) (*MinioServer, error) { msg("running minio server\n") cfgdir, err := ioutil.TempDir("", "minio-config-") if err != nil { @@ -304,7 +349,7 @@ func NewMinioServer() (*MinioServer, error) { out := bytes.NewBuffer(nil) - cmd := exec.Command("minio", + cmd := exec.Command(minio, "--config-folder", cfgdir, "--address", "127.0.0.1:9000", "server", dir) From 3d06e6083a63e4b06ace38ae9ede61c1e533c1f6 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 26 Jan 2016 23:52:39 +0100 Subject: [PATCH 55/56] CI: download minio for the correct os and arch --- Dockerfile | 2 +- run_integration_tests.go | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7b5a94cd4..5e6f91638 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,7 +48,7 @@ RUN go get github.com/mattn/goveralls RUN go get github.com/mitchellh/gox RUN go get github.com/pierrre/gotestcover RUN mkdir $HOME/bin \ - && wget -q -O $HOME/bin/minio https://dl.minio.io/server/minio/release/linux-amd64/minio \ + && wget -q -O $HOME/bin/minio https://dl.minio.io/server/minio/release/linux-${GOARCH}/minio \ && chmod +x $HOME/bin/minio # set TRAVIS_BUILD_DIR for integration script diff --git a/run_integration_tests.go b/run_integration_tests.go index 0a4c59401..a771e274a 100644 --- a/run_integration_tests.go +++ b/run_integration_tests.go @@ -49,7 +49,10 @@ func (env *TravisEnvironment) getMinio() { os.Exit(10) } - res, err := http.Get("https://dl.minio.io/server/minio/release/linux-amd64/minio") + url := fmt.Sprintf("https://dl.minio.io/server/minio/release/%s-%s/minio", + runtime.GOOS, runtime.GOARCH) + msg("downloading %v\n", url) + res, err := http.Get(url) if err != nil { msg("downloading minio failed: %v\n", err) return From 322eca86bcb5976ebcedd814a61c39df1f5122c0 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 27 Jan 2016 21:33:48 +0100 Subject: [PATCH 56/56] mem backend: remove unused code --- backend/mem/mem_backend.go | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/backend/mem/mem_backend.go b/backend/mem/mem_backend.go index 4e8d3b46c..adac5b332 100644 --- a/backend/mem/mem_backend.go +++ b/backend/mem/mem_backend.go @@ -1,7 +1,6 @@ package mem import ( - "bytes" "errors" "io" "sync" @@ -98,30 +97,6 @@ func memTest(be *MemoryBackend, t backend.Type, name string) (bool, error) { return false, nil } -// tempMemEntry temporarily holds data written to the memory backend before it -// is finalized. -type tempMemEntry struct { - be *MemoryBackend - data bytes.Buffer -} - -func (e *tempMemEntry) Write(p []byte) (int, error) { - return e.data.Write(p) -} - -func (e *tempMemEntry) Size() uint { - return uint(len(e.data.Bytes())) -} - -func (e *tempMemEntry) Finalize(t backend.Type, name string) error { - if t == backend.Config { - name = "" - } - - debug.Log("MemoryBackend", "save blob %p (%d bytes) as %v %v", e, len(e.data.Bytes()), t, name) - return e.be.insert(t, name, e.data.Bytes()) -} - func memLoad(be *MemoryBackend, h backend.Handle, p []byte, off int64) (int, error) { if err := h.Valid(); err != nil { return 0, err