From b8b5c8e8c9624d86961863dc389a1aa583d381b2 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 7 Jun 2017 21:59:01 +0200 Subject: [PATCH 1/6] s3: Rename struct to Backend --- src/restic/backend/s3/s3.go | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index 767c21dc0..5a1aefd11 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -21,8 +21,8 @@ import ( const connLimit = 10 -// s3 is a backend which stores the data on an S3 endpoint. -type s3 struct { +// Backend stores data on an S3 endpoint. +type Backend struct { client *minio.Client sem *backend.Semaphore bucketname string @@ -32,8 +32,8 @@ type s3 struct { backend.Layout } -// make sure that *s3 implements backend.Backend -var _ restic.Backend = &s3{} +// make sure that *Backend implements backend.Backend +var _ restic.Backend = &Backend{} const defaultLayout = "s3legacy" @@ -52,7 +52,7 @@ func Open(cfg Config) (restic.Backend, error) { return nil, err } - be := &s3{ + be := &Backend{ client: client, sem: sem, bucketname: cfg.Bucket, @@ -87,13 +87,13 @@ func Open(cfg Config) (restic.Backend, error) { } // IsNotExist returns true if the error is caused by a not existing file. -func (be *s3) IsNotExist(err error) bool { +func (be *Backend) IsNotExist(err error) bool { debug.Log("IsNotExist(%T, %#v)", err, err) return os.IsNotExist(err) } // Join combines path components with slashes. -func (be *s3) Join(p ...string) string { +func (be *Backend) Join(p ...string) string { return path.Join(p...) } @@ -113,7 +113,7 @@ func (fi fileInfo) IsDir() bool { return fi.isDir } // abbreviation for func (fi fileInfo) Sys() interface{} { return nil } // underlying data source (can return nil) // ReadDir returns the entries for a directory. -func (be *s3) ReadDir(dir string) (list []os.FileInfo, err error) { +func (be *Backend) ReadDir(dir string) (list []os.FileInfo, err error) { debug.Log("ReadDir(%v)", dir) // make sure dir ends with a slash @@ -154,7 +154,7 @@ func (be *s3) ReadDir(dir string) (list []os.FileInfo, err error) { } // Location returns this backend's location (the bucket name). -func (be *s3) Location() string { +func (be *Backend) Location() string { return be.bucketname } @@ -203,7 +203,7 @@ func (wr preventCloser) Close() error { } // Save stores data in the backend at the handle. -func (be *s3) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) { +func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) { if err := h.Valid(); err != nil { return err } @@ -259,7 +259,7 @@ func (wr wrapReader) Close() error { // Load returns a reader that yields the contents of the file at h at the // given offset. If length is nonzero, only a portion of the file is // returned. rd must be closed after use. -func (be *s3) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { +func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h)) if err := h.Valid(); err != nil { return nil, err @@ -304,7 +304,7 @@ func (be *s3) Load(ctx context.Context, h restic.Handle, length int, offset int6 } // Stat returns information about a blob. -func (be *s3) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { +func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { debug.Log("%v", h) objName := be.Filename(h) @@ -334,7 +334,7 @@ func (be *s3) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, er } // Test returns true if a blob of the given type and name exists in the backend. -func (be *s3) Test(ctx context.Context, h restic.Handle) (bool, error) { +func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) { found := false objName := be.Filename(h) _, err := be.client.StatObject(be.bucketname, objName) @@ -347,7 +347,7 @@ func (be *s3) Test(ctx context.Context, h restic.Handle) (bool, error) { } // Remove removes the blob with the given name and type. -func (be *s3) Remove(ctx context.Context, h restic.Handle) error { +func (be *Backend) Remove(ctx context.Context, h restic.Handle) error { objName := be.Filename(h) err := be.client.RemoveObject(be.bucketname, objName) debug.Log("Remove(%v) at %v -> err %v", h, objName, err) @@ -357,7 +357,7 @@ func (be *s3) Remove(ctx context.Context, h restic.Handle) error { // List returns a channel that yields all names of blobs of type t. A // goroutine is started for this. If the channel done is closed, sending // stops. -func (be *s3) List(ctx context.Context, t restic.FileType) <-chan string { +func (be *Backend) List(ctx context.Context, t restic.FileType) <-chan string { debug.Log("listing %v", t) ch := make(chan string) @@ -390,7 +390,7 @@ func (be *s3) List(ctx context.Context, t restic.FileType) <-chan string { } // Remove keys for a specified backend type. -func (be *s3) removeKeys(ctx context.Context, t restic.FileType) error { +func (be *Backend) removeKeys(ctx context.Context, t restic.FileType) error { for key := range be.List(ctx, restic.DataFile) { err := be.Remove(ctx, restic.Handle{Type: restic.DataFile, Name: key}) if err != nil { @@ -402,7 +402,7 @@ func (be *s3) removeKeys(ctx context.Context, t restic.FileType) error { } // Delete removes all restic keys in the bucket. It will not remove the bucket itself. -func (be *s3) Delete(ctx context.Context) error { +func (be *Backend) Delete(ctx context.Context) error { alltypes := []restic.FileType{ restic.DataFile, restic.KeyFile, @@ -421,4 +421,4 @@ func (be *s3) Delete(ctx context.Context) error { } // Close does nothing -func (be *s3) Close() error { return nil } +func (be *Backend) Close() error { return nil } From 7cf8f59987889839858e2ecaf418f839d934612c Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 7 Jun 2017 21:59:41 +0200 Subject: [PATCH 2/6] layout: Add String() and Name() --- src/restic/backend/layout.go | 1 + src/restic/backend/layout_default.go | 9 +++++++++ src/restic/backend/layout_rest.go | 9 +++++++++ src/restic/backend/layout_s3legacy.go | 9 +++++++++ 4 files changed, 28 insertions(+) diff --git a/src/restic/backend/layout.go b/src/restic/backend/layout.go index 3d0953de8..b58f290be 100644 --- a/src/restic/backend/layout.go +++ b/src/restic/backend/layout.go @@ -17,6 +17,7 @@ type Layout interface { Dirname(restic.Handle) string Basedir(restic.FileType) string Paths() []string + Name() string } // Filesystem is the abstraction of a file system used for a backend. diff --git a/src/restic/backend/layout_default.go b/src/restic/backend/layout_default.go index 665b85531..9b57657b5 100644 --- a/src/restic/backend/layout_default.go +++ b/src/restic/backend/layout_default.go @@ -19,6 +19,15 @@ var defaultLayoutPaths = map[restic.FileType]string{ restic.KeyFile: "keys", } +func (l *DefaultLayout) String() string { + return "" +} + +// Name returns the name for this layout. +func (l *DefaultLayout) Name() string { + return "default" +} + // Dirname returns the directory path for a given file type and name. func (l *DefaultLayout) Dirname(h restic.Handle) string { p := defaultLayoutPaths[h.Type] diff --git a/src/restic/backend/layout_rest.go b/src/restic/backend/layout_rest.go index 2d01ece79..007be37c8 100644 --- a/src/restic/backend/layout_rest.go +++ b/src/restic/backend/layout_rest.go @@ -11,6 +11,15 @@ type RESTLayout struct { var restLayoutPaths = defaultLayoutPaths +func (l *RESTLayout) String() string { + return "" +} + +// Name returns the name for this layout. +func (l *RESTLayout) Name() string { + return "rest" +} + // Dirname returns the directory path for a given file type and name. func (l *RESTLayout) Dirname(h restic.Handle) string { if h.Type == restic.ConfigFile { diff --git a/src/restic/backend/layout_s3legacy.go b/src/restic/backend/layout_s3legacy.go index 601d29bc5..d9d7ab212 100644 --- a/src/restic/backend/layout_s3legacy.go +++ b/src/restic/backend/layout_s3legacy.go @@ -18,6 +18,15 @@ var s3LayoutPaths = map[restic.FileType]string{ restic.KeyFile: "key", } +func (l *S3LegacyLayout) String() string { + return "" +} + +// Name returns the name for this layout. +func (l *S3LegacyLayout) Name() string { + return "s3legacy" +} + // join calls Join with the first empty elements removed. func (l *S3LegacyLayout) join(url string, items ...string) string { for len(items) > 0 && items[0] == "" { From 4f9bf5312b6dc8269632e59f00bcc63666cef77f Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 7 Jun 2017 22:54:37 +0200 Subject: [PATCH 3/6] Add migrate This commits adds a 'migrate' command and a migration to move s3 repositories from the 's3legacy' to the 'default' layout. --- src/cmds/restic/cmd_migrate.go | 99 ++++++++++++++++++++++++++++++ src/restic/backend/s3/s3.go | 25 +++++++- src/restic/migrations/doc.go | 2 + src/restic/migrations/interface.go | 21 +++++++ src/restic/migrations/list.go | 8 +++ src/restic/migrations/s3_layout.go | 87 ++++++++++++++++++++++++++ 6 files changed, 241 insertions(+), 1 deletion(-) create mode 100644 src/cmds/restic/cmd_migrate.go create mode 100644 src/restic/migrations/doc.go create mode 100644 src/restic/migrations/interface.go create mode 100644 src/restic/migrations/list.go create mode 100644 src/restic/migrations/s3_layout.go diff --git a/src/cmds/restic/cmd_migrate.go b/src/cmds/restic/cmd_migrate.go new file mode 100644 index 000000000..6be04cc0b --- /dev/null +++ b/src/cmds/restic/cmd_migrate.go @@ -0,0 +1,99 @@ +package main + +import ( + "restic" + "restic/migrations" + + "github.com/spf13/cobra" +) + +var cmdMigrate = &cobra.Command{ + Use: "migrate [name]", + Short: "apply migrations", + Long: ` +The "migrate" command applies migrations to a repository. When no migration +name is explicitely given, a list of migrations that can be applied is printed. +`, + RunE: func(cmd *cobra.Command, args []string) error { + return runMigrate(migrateOptions, globalOptions, args) + }, +} + +// MigrateOptions bundles all options for the 'check' command. +type MigrateOptions struct { +} + +var migrateOptions MigrateOptions + +func init() { + cmdRoot.AddCommand(cmdMigrate) +} + +func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository) error { + ctx := gopts.ctx + Printf("available migrations:\n") + for _, m := range migrations.All { + ok, err := m.Check(ctx, repo) + if err != nil { + return err + } + + if ok { + Printf(" %v\n", m.Name()) + } + } + + return nil +} + +func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string) error { + ctx := gopts.ctx + + var firsterr error + for _, name := range args { + for _, m := range migrations.All { + if m.Name() == name { + ok, err := m.Check(ctx, repo) + if err != nil { + return err + } + + if !ok { + Warnf("migration %v cannot be applied: check failed\n") + continue + } + + if err = m.Apply(ctx, repo); err != nil { + Warnf("migration %v failed: %v\n", m.Name(), err) + if firsterr == nil { + firsterr = err + } + continue + } + + Printf("migration %v: success\n", m.Name()) + } + } + } + + return firsterr +} + +func runMigrate(opts MigrateOptions, gopts GlobalOptions, args []string) error { + repo, err := OpenRepository(gopts) + if err != nil { + return err + } + + lock, err := lockRepoExclusive(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + + if len(args) == 0 { + return checkMigrations(opts, gopts, repo) + } + + return applyMigrations(opts, gopts, repo, args) +} diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index 5a1aefd11..ef7908786 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -155,7 +155,12 @@ func (be *Backend) ReadDir(dir string) (list []os.FileInfo, err error) { // Location returns this backend's location (the bucket name). func (be *Backend) Location() string { - return be.bucketname + return be.Join(be.bucketname, be.prefix) +} + +// Path returns the path in the bucket that is used for this backend. +func (be *Backend) Path() string { + return be.prefix } // getRemainingSize returns number of bytes remaining. If it is not possible to @@ -422,3 +427,21 @@ func (be *Backend) Delete(ctx context.Context) error { // Close does nothing func (be *Backend) Close() error { return nil } + +// Rename moves a file based on the new layout l. +func (be *Backend) Rename(h restic.Handle, l backend.Layout) error { + debug.Log("Rename %v to %v", h, l) + oldname := be.Filename(h) + newname := l.Filename(h) + + debug.Log(" %v -> %v", oldname, newname) + + coreClient := minio.Core{Client: be.client} + err := coreClient.CopyObject(be.bucketname, newname, path.Join(be.bucketname, oldname), minio.CopyConditions{}) + if err != nil { + debug.Log("copy failed: %v", err) + return err + } + + return be.client.RemoveObject(be.bucketname, oldname) +} diff --git a/src/restic/migrations/doc.go b/src/restic/migrations/doc.go new file mode 100644 index 000000000..0c757fcf4 --- /dev/null +++ b/src/restic/migrations/doc.go @@ -0,0 +1,2 @@ +// Package migrations contains migrations that can be applied to a repository and/or backend. +package migrations diff --git a/src/restic/migrations/interface.go b/src/restic/migrations/interface.go new file mode 100644 index 000000000..288ca273b --- /dev/null +++ b/src/restic/migrations/interface.go @@ -0,0 +1,21 @@ +package migrations + +import ( + "context" + "restic" +) + +// Migration implements a data migration. +type Migration interface { + // Check returns true if the migration can be applied to a repo. + Check(context.Context, restic.Repository) (bool, error) + + // Apply runs the migration. + Apply(context.Context, restic.Repository) error + + // Name returns a short name. + Name() string + + // Descr returns a description what the migration does. + Desc() string +} diff --git a/src/restic/migrations/list.go b/src/restic/migrations/list.go new file mode 100644 index 000000000..4442f343c --- /dev/null +++ b/src/restic/migrations/list.go @@ -0,0 +1,8 @@ +package migrations + +// All contains all migrations. +var All []Migration + +func register(m Migration) { + All = append(All, m) +} diff --git a/src/restic/migrations/s3_layout.go b/src/restic/migrations/s3_layout.go new file mode 100644 index 000000000..75a83b885 --- /dev/null +++ b/src/restic/migrations/s3_layout.go @@ -0,0 +1,87 @@ +package migrations + +import ( + "context" + "path" + "restic" + "restic/backend" + "restic/backend/s3" + "restic/debug" + "restic/errors" +) + +func init() { + register(&S3Layout{}) +} + +// S3Layout migrates a repository on an S3 backend from the "s3legacy" to the +// "default" layout. +type S3Layout struct{} + +// Check tests whether the migration can be applied. +func (m *S3Layout) Check(ctx context.Context, repo restic.Repository) (bool, error) { + be, ok := repo.Backend().(*s3.Backend) + if !ok { + debug.Log("backend is not s3") + return false, nil + } + + if be.Layout.Name() != "s3legacy" { + debug.Log("layout is not s3legacy") + return false, nil + } + + return true, nil +} + +func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l backend.Layout, t restic.FileType) error { + for name := range be.List(ctx, t) { + h := restic.Handle{Type: t, Name: name} + debug.Log("move %v", h) + if err := be.Rename(h, l); err != nil { + return err + } + } + + return nil +} + +// Apply runs the migration. +func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error { + be, ok := repo.Backend().(*s3.Backend) + if !ok { + debug.Log("backend is not s3") + return errors.New("backend is not s3") + } + + newLayout := &backend.DefaultLayout{ + Path: be.Path(), + Join: path.Join, + } + + for _, t := range []restic.FileType{ + restic.KeyFile, + restic.SnapshotFile, + restic.DataFile, + restic.LockFile, + } { + err := m.moveFiles(ctx, be, newLayout, t) + if err != nil { + return err + } + } + + be.Layout = newLayout + + return nil +} + +// Name returns the name for this migration. +func (m *S3Layout) Name() string { + return "s3_layout" +} + +// Desc returns a short description what the migration does. +func (m *S3Layout) Desc() string { + return "move files from 's3legacy' to the 'default' repository layout" +} From 04ded881f67df236eafe19a9ed4e0074e5f760e5 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 7 Jun 2017 23:08:20 +0200 Subject: [PATCH 4/6] s3: Change the default layout to "default" --- src/restic/backend/s3/s3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/restic/backend/s3/s3.go b/src/restic/backend/s3/s3.go index ef7908786..65c727613 100644 --- a/src/restic/backend/s3/s3.go +++ b/src/restic/backend/s3/s3.go @@ -35,7 +35,7 @@ type Backend struct { // make sure that *Backend implements backend.Backend var _ restic.Backend = &Backend{} -const defaultLayout = "s3legacy" +const defaultLayout = "default" // Open opens the S3 backend at bucket and region. The bucket is created if it // does not exist yet. From ff3d2e42f472e6aedbb1773e069125d96bdabbba Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Thu, 8 Jun 2017 19:19:45 +0200 Subject: [PATCH 5/6] migrate: Be a bit more verbose --- src/cmds/restic/cmd_migrate.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/cmds/restic/cmd_migrate.go b/src/cmds/restic/cmd_migrate.go index 6be04cc0b..b585534a2 100644 --- a/src/cmds/restic/cmd_migrate.go +++ b/src/cmds/restic/cmd_migrate.go @@ -39,7 +39,7 @@ func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos } if ok { - Printf(" %v\n", m.Name()) + Printf(" %v: %v\n", m.Name(), m.Desc()) } } @@ -59,10 +59,11 @@ func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repos } if !ok { - Warnf("migration %v cannot be applied: check failed\n") + Warnf("migration %v cannot be applied: check failed\n", m.Name()) continue } + Printf("applying migration %v...\n", m.Name()) if err = m.Apply(ctx, repo); err != nil { Warnf("migration %v failed: %v\n", m.Name(), err) if firsterr == nil { From eb7ddd6e11d9971d77f0dac7430fbea6807cd95d Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Thu, 8 Jun 2017 19:21:52 +0200 Subject: [PATCH 6/6] Add entry to CHANGELOG --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 92274a1be..801168bbf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,12 @@ Important Changes in 0.X.Y https://github.com/restic/restic/issues/989 https://github.com/restic/restic/pull/993 + * The default layout for the s3 backend is now `default` (instead of + `s3legacy`). Also, there's a new `migrate` command to convert an existing + repo, it can be run like this: `restic migrate s3_layout` + https://github.com/restic/restic/issues/965 + https://github.com/restic/restic/pull/1004 + Important Changes in 0.6.1 ==========================