Merge pull request #1004 from restic/add-migrate-s3

Add 'migrate' command, change s3 layout
This commit is contained in:
Alexander Neumann 2017-06-08 20:48:27 +02:00
commit 1f0916b01b
11 changed files with 295 additions and 20 deletions

View File

@ -24,6 +24,12 @@ Important Changes in 0.X.Y
large files now is significantly faster. large files now is significantly faster.
https://github.com/restic/restic/pull/998 https://github.com/restic/restic/pull/998
* The default layout for the s3 backend is now `default` (instead of
`s3legacy`). Also, there's a new `migrate` command to convert an existing
repo, it can be run like this: `restic migrate s3_layout`
https://github.com/restic/restic/issues/965
https://github.com/restic/restic/pull/1004
Important Changes in 0.6.1 Important Changes in 0.6.1
========================== ==========================

View File

@ -0,0 +1,100 @@
package main
import (
"restic"
"restic/migrations"
"github.com/spf13/cobra"
)
var cmdMigrate = &cobra.Command{
Use: "migrate [name]",
Short: "apply migrations",
Long: `
The "migrate" command applies migrations to a repository. When no migration
name is explicitely given, a list of migrations that can be applied is printed.
`,
RunE: func(cmd *cobra.Command, args []string) error {
return runMigrate(migrateOptions, globalOptions, args)
},
}
// MigrateOptions bundles all options for the 'check' command.
type MigrateOptions struct {
}
var migrateOptions MigrateOptions
func init() {
cmdRoot.AddCommand(cmdMigrate)
}
func checkMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository) error {
ctx := gopts.ctx
Printf("available migrations:\n")
for _, m := range migrations.All {
ok, err := m.Check(ctx, repo)
if err != nil {
return err
}
if ok {
Printf(" %v: %v\n", m.Name(), m.Desc())
}
}
return nil
}
func applyMigrations(opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string) error {
ctx := gopts.ctx
var firsterr error
for _, name := range args {
for _, m := range migrations.All {
if m.Name() == name {
ok, err := m.Check(ctx, repo)
if err != nil {
return err
}
if !ok {
Warnf("migration %v cannot be applied: check failed\n", m.Name())
continue
}
Printf("applying migration %v...\n", m.Name())
if err = m.Apply(ctx, repo); err != nil {
Warnf("migration %v failed: %v\n", m.Name(), err)
if firsterr == nil {
firsterr = err
}
continue
}
Printf("migration %v: success\n", m.Name())
}
}
}
return firsterr
}
func runMigrate(opts MigrateOptions, gopts GlobalOptions, args []string) error {
repo, err := OpenRepository(gopts)
if err != nil {
return err
}
lock, err := lockRepoExclusive(repo)
defer unlockRepo(lock)
if err != nil {
return err
}
if len(args) == 0 {
return checkMigrations(opts, gopts, repo)
}
return applyMigrations(opts, gopts, repo, args)
}

View File

@ -17,6 +17,7 @@ type Layout interface {
Dirname(restic.Handle) string Dirname(restic.Handle) string
Basedir(restic.FileType) string Basedir(restic.FileType) string
Paths() []string Paths() []string
Name() string
} }
// Filesystem is the abstraction of a file system used for a backend. // Filesystem is the abstraction of a file system used for a backend.

View File

@ -19,6 +19,15 @@ var defaultLayoutPaths = map[restic.FileType]string{
restic.KeyFile: "keys", restic.KeyFile: "keys",
} }
func (l *DefaultLayout) String() string {
return "<DefaultLayout>"
}
// Name returns the name for this layout.
func (l *DefaultLayout) Name() string {
return "default"
}
// Dirname returns the directory path for a given file type and name. // Dirname returns the directory path for a given file type and name.
func (l *DefaultLayout) Dirname(h restic.Handle) string { func (l *DefaultLayout) Dirname(h restic.Handle) string {
p := defaultLayoutPaths[h.Type] p := defaultLayoutPaths[h.Type]

View File

@ -11,6 +11,15 @@ type RESTLayout struct {
var restLayoutPaths = defaultLayoutPaths var restLayoutPaths = defaultLayoutPaths
func (l *RESTLayout) String() string {
return "<RESTLayout>"
}
// Name returns the name for this layout.
func (l *RESTLayout) Name() string {
return "rest"
}
// Dirname returns the directory path for a given file type and name. // Dirname returns the directory path for a given file type and name.
func (l *RESTLayout) Dirname(h restic.Handle) string { func (l *RESTLayout) Dirname(h restic.Handle) string {
if h.Type == restic.ConfigFile { if h.Type == restic.ConfigFile {

View File

@ -18,6 +18,15 @@ var s3LayoutPaths = map[restic.FileType]string{
restic.KeyFile: "key", restic.KeyFile: "key",
} }
func (l *S3LegacyLayout) String() string {
return "<S3LegacyLayout>"
}
// Name returns the name for this layout.
func (l *S3LegacyLayout) Name() string {
return "s3legacy"
}
// join calls Join with the first empty elements removed. // join calls Join with the first empty elements removed.
func (l *S3LegacyLayout) join(url string, items ...string) string { func (l *S3LegacyLayout) join(url string, items ...string) string {
for len(items) > 0 && items[0] == "" { for len(items) > 0 && items[0] == "" {

View File

@ -20,8 +20,8 @@ import (
const connLimit = 10 const connLimit = 10
// s3 is a backend which stores the data on an S3 endpoint. // Backend stores data on an S3 endpoint.
type s3 struct { type Backend struct {
client *minio.Client client *minio.Client
sem *backend.Semaphore sem *backend.Semaphore
bucketname string bucketname string
@ -29,10 +29,10 @@ type s3 struct {
backend.Layout backend.Layout
} }
// make sure that *s3 implements backend.Backend // make sure that *Backend implements backend.Backend
var _ restic.Backend = &s3{} var _ restic.Backend = &Backend{}
const defaultLayout = "s3legacy" const defaultLayout = "default"
// Open opens the S3 backend at bucket and region. The bucket is created if it // Open opens the S3 backend at bucket and region. The bucket is created if it
// does not exist yet. // does not exist yet.
@ -49,7 +49,7 @@ func Open(cfg Config) (restic.Backend, error) {
return nil, err return nil, err
} }
be := &s3{ be := &Backend{
client: client, client: client,
sem: sem, sem: sem,
bucketname: cfg.Bucket, bucketname: cfg.Bucket,
@ -83,13 +83,13 @@ func Open(cfg Config) (restic.Backend, error) {
} }
// IsNotExist returns true if the error is caused by a not existing file. // IsNotExist returns true if the error is caused by a not existing file.
func (be *s3) IsNotExist(err error) bool { func (be *Backend) IsNotExist(err error) bool {
debug.Log("IsNotExist(%T, %#v)", err, err) debug.Log("IsNotExist(%T, %#v)", err, err)
return os.IsNotExist(err) return os.IsNotExist(err)
} }
// Join combines path components with slashes. // Join combines path components with slashes.
func (be *s3) Join(p ...string) string { func (be *Backend) Join(p ...string) string {
return path.Join(p...) return path.Join(p...)
} }
@ -109,7 +109,7 @@ func (fi fileInfo) IsDir() bool { return fi.isDir } // abbreviation for
func (fi fileInfo) Sys() interface{} { return nil } // underlying data source (can return nil) func (fi fileInfo) Sys() interface{} { return nil } // underlying data source (can return nil)
// ReadDir returns the entries for a directory. // ReadDir returns the entries for a directory.
func (be *s3) ReadDir(dir string) (list []os.FileInfo, err error) { func (be *Backend) ReadDir(dir string) (list []os.FileInfo, err error) {
debug.Log("ReadDir(%v)", dir) debug.Log("ReadDir(%v)", dir)
// make sure dir ends with a slash // make sure dir ends with a slash
@ -150,8 +150,13 @@ func (be *s3) ReadDir(dir string) (list []os.FileInfo, err error) {
} }
// Location returns this backend's location (the bucket name). // Location returns this backend's location (the bucket name).
func (be *s3) Location() string { func (be *Backend) Location() string {
return be.bucketname return be.Join(be.bucketname, be.prefix)
}
// Path returns the path in the bucket that is used for this backend.
func (be *Backend) Path() string {
return be.prefix
} }
// getRemainingSize returns number of bytes remaining. If it is not possible to // getRemainingSize returns number of bytes remaining. If it is not possible to
@ -199,7 +204,7 @@ func (wr preventCloser) Close() error {
} }
// Save stores data in the backend at the handle. // Save stores data in the backend at the handle.
func (be *s3) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) { func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
if err := h.Valid(); err != nil { if err := h.Valid(); err != nil {
return err return err
} }
@ -255,7 +260,7 @@ func (wr wrapReader) Close() error {
// Load returns a reader that yields the contents of the file at h at the // Load returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is // given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use. // returned. rd must be closed after use.
func (be *s3) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) { func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h)) debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h))
if err := h.Valid(); err != nil { if err := h.Valid(); err != nil {
return nil, err return nil, err
@ -300,7 +305,7 @@ func (be *s3) Load(ctx context.Context, h restic.Handle, length int, offset int6
} }
// Stat returns information about a blob. // Stat returns information about a blob.
func (be *s3) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) { func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
debug.Log("%v", h) debug.Log("%v", h)
objName := be.Filename(h) objName := be.Filename(h)
@ -330,7 +335,7 @@ func (be *s3) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, er
} }
// Test returns true if a blob of the given type and name exists in the backend. // Test returns true if a blob of the given type and name exists in the backend.
func (be *s3) Test(ctx context.Context, h restic.Handle) (bool, error) { func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
found := false found := false
objName := be.Filename(h) objName := be.Filename(h)
_, err := be.client.StatObject(be.bucketname, objName) _, err := be.client.StatObject(be.bucketname, objName)
@ -343,7 +348,7 @@ func (be *s3) Test(ctx context.Context, h restic.Handle) (bool, error) {
} }
// Remove removes the blob with the given name and type. // Remove removes the blob with the given name and type.
func (be *s3) Remove(ctx context.Context, h restic.Handle) error { func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
objName := be.Filename(h) objName := be.Filename(h)
err := be.client.RemoveObject(be.bucketname, objName) err := be.client.RemoveObject(be.bucketname, objName)
debug.Log("Remove(%v) at %v -> err %v", h, objName, err) debug.Log("Remove(%v) at %v -> err %v", h, objName, err)
@ -353,7 +358,7 @@ func (be *s3) Remove(ctx context.Context, h restic.Handle) error {
// List returns a channel that yields all names of blobs of type t. A // List returns a channel that yields all names of blobs of type t. A
// goroutine is started for this. If the channel done is closed, sending // goroutine is started for this. If the channel done is closed, sending
// stops. // stops.
func (be *s3) List(ctx context.Context, t restic.FileType) <-chan string { func (be *Backend) List(ctx context.Context, t restic.FileType) <-chan string {
debug.Log("listing %v", t) debug.Log("listing %v", t)
ch := make(chan string) ch := make(chan string)
@ -386,7 +391,7 @@ func (be *s3) List(ctx context.Context, t restic.FileType) <-chan string {
} }
// Remove keys for a specified backend type. // Remove keys for a specified backend type.
func (be *s3) removeKeys(ctx context.Context, t restic.FileType) error { func (be *Backend) removeKeys(ctx context.Context, t restic.FileType) error {
for key := range be.List(ctx, restic.DataFile) { for key := range be.List(ctx, restic.DataFile) {
err := be.Remove(ctx, restic.Handle{Type: restic.DataFile, Name: key}) err := be.Remove(ctx, restic.Handle{Type: restic.DataFile, Name: key})
if err != nil { if err != nil {
@ -398,7 +403,7 @@ func (be *s3) removeKeys(ctx context.Context, t restic.FileType) error {
} }
// Delete removes all restic keys in the bucket. It will not remove the bucket itself. // Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *s3) Delete(ctx context.Context) error { func (be *Backend) Delete(ctx context.Context) error {
alltypes := []restic.FileType{ alltypes := []restic.FileType{
restic.DataFile, restic.DataFile,
restic.KeyFile, restic.KeyFile,
@ -417,4 +422,22 @@ func (be *s3) Delete(ctx context.Context) error {
} }
// Close does nothing // Close does nothing
func (be *s3) Close() error { return nil } func (be *Backend) Close() error { return nil }
// Rename moves a file based on the new layout l.
func (be *Backend) Rename(h restic.Handle, l backend.Layout) error {
debug.Log("Rename %v to %v", h, l)
oldname := be.Filename(h)
newname := l.Filename(h)
debug.Log(" %v -> %v", oldname, newname)
coreClient := minio.Core{Client: be.client}
err := coreClient.CopyObject(be.bucketname, newname, path.Join(be.bucketname, oldname), minio.CopyConditions{})
if err != nil {
debug.Log("copy failed: %v", err)
return err
}
return be.client.RemoveObject(be.bucketname, oldname)
}

View File

@ -0,0 +1,2 @@
// Package migrations contains migrations that can be applied to a repository and/or backend.
package migrations

View File

@ -0,0 +1,21 @@
package migrations
import (
"context"
"restic"
)
// Migration implements a data migration.
type Migration interface {
// Check returns true if the migration can be applied to a repo.
Check(context.Context, restic.Repository) (bool, error)
// Apply runs the migration.
Apply(context.Context, restic.Repository) error
// Name returns a short name.
Name() string
// Descr returns a description what the migration does.
Desc() string
}

View File

@ -0,0 +1,8 @@
package migrations
// All contains all migrations.
var All []Migration
func register(m Migration) {
All = append(All, m)
}

View File

@ -0,0 +1,87 @@
package migrations
import (
"context"
"path"
"restic"
"restic/backend"
"restic/backend/s3"
"restic/debug"
"restic/errors"
)
func init() {
register(&S3Layout{})
}
// S3Layout migrates a repository on an S3 backend from the "s3legacy" to the
// "default" layout.
type S3Layout struct{}
// Check tests whether the migration can be applied.
func (m *S3Layout) Check(ctx context.Context, repo restic.Repository) (bool, error) {
be, ok := repo.Backend().(*s3.Backend)
if !ok {
debug.Log("backend is not s3")
return false, nil
}
if be.Layout.Name() != "s3legacy" {
debug.Log("layout is not s3legacy")
return false, nil
}
return true, nil
}
func (m *S3Layout) moveFiles(ctx context.Context, be *s3.Backend, l backend.Layout, t restic.FileType) error {
for name := range be.List(ctx, t) {
h := restic.Handle{Type: t, Name: name}
debug.Log("move %v", h)
if err := be.Rename(h, l); err != nil {
return err
}
}
return nil
}
// Apply runs the migration.
func (m *S3Layout) Apply(ctx context.Context, repo restic.Repository) error {
be, ok := repo.Backend().(*s3.Backend)
if !ok {
debug.Log("backend is not s3")
return errors.New("backend is not s3")
}
newLayout := &backend.DefaultLayout{
Path: be.Path(),
Join: path.Join,
}
for _, t := range []restic.FileType{
restic.KeyFile,
restic.SnapshotFile,
restic.DataFile,
restic.LockFile,
} {
err := m.moveFiles(ctx, be, newLayout, t)
if err != nil {
return err
}
}
be.Layout = newLayout
return nil
}
// Name returns the name for this migration.
func (m *S3Layout) Name() string {
return "s3_layout"
}
// Desc returns a short description what the migration does.
func (m *S3Layout) Desc() string {
return "move files from 's3legacy' to the 'default' repository layout"
}