Merge pull request #509 from restic/read-from-stdin

Allow reading data from stdin
This commit is contained in:
Alexander Neumann 2016-05-09 22:12:22 +02:00
commit 2c1e590e47
7 changed files with 356 additions and 4 deletions

View File

@ -182,6 +182,22 @@ see [`filepath.Match`](https://golang.org/pkg/path/filepath/#Match) for syntax.
Additionally `**` exludes arbitrary subdirectories.
Environment-variables in exclude-files are expanded with [`os.ExpandEnv`](https://golang.org/pkg/os/#ExpandEnv).
## Reading data from stdin
Sometimes it can be nice to directly save the output of a program, e.g.
`mysqldump` so that the SQL can later be restored. Restic supports this mode of
operation, just supply the option `--stdin` to the `backup` command like this:
$ mysqldump [...] | restic -r /tmp/backup backup --stdin
This creates a new snapshot of the output of `mqsqldump`. You can then use e.g.
the fuse mounting option (see below) to mount the repository and read the file.
By default, the file name `stdin` is used, a different name can be specified
with `--stdin-filename`, e.g. like this:
$ mysqldump [...] | restic -r /tmp/backup backup --stdin --stdin-filenam production.sql
# List all snapshots
Now, you can list all the snapshots stored in the repository:

View File

@ -18,10 +18,12 @@ import (
)
type CmdBackup struct {
Parent string `short:"p" long:"parent" description:"use this parent snapshot (default: last snapshot in repo that has the same target)"`
Force bool `short:"f" long:"force" description:"Force re-reading the target. Overrides the \"parent\" flag"`
Excludes []string `short:"e" long:"exclude" description:"Exclude a pattern (can be specified multiple times)"`
ExcludeFile string `long:"exclude-file" description:"Read exclude-patterns from file"`
Parent string `short:"p" long:"parent" description:"use this parent snapshot (default: last snapshot in repo that has the same target)"`
Force bool `short:"f" long:"force" description:"Force re-reading the target. Overrides the \"parent\" flag"`
Excludes []string `short:"e" long:"exclude" description:"Exclude a pattern (can be specified multiple times)"`
ExcludeFile string `long:"exclude-file" description:"Read exclude-patterns from file"`
Stdin bool `long:"stdin" description:"read backup data from stdin"`
StdinFilename string `long:"stdin-filename" default:"stdin" description:"file name to use when reading from stdin"`
global *GlobalOptions
}
@ -175,6 +177,47 @@ func (cmd CmdBackup) newArchiveProgress(todo restic.Stat) *restic.Progress {
return archiveProgress
}
func (cmd CmdBackup) newArchiveStdinProgress() *restic.Progress {
if !cmd.global.ShowProgress() {
return nil
}
archiveProgress := restic.NewProgress(time.Second)
var bps uint64
archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
sec := uint64(d / time.Second)
if s.Bytes > 0 && sec > 0 && ticker {
bps = s.Bytes / sec
}
status1 := fmt.Sprintf("[%s] %s %s/s", formatDuration(d),
formatBytes(s.Bytes),
formatBytes(bps))
w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
if err == nil {
maxlen := w - len(status1)
if maxlen < 4 {
status1 = ""
} else if len(status1) > maxlen {
status1 = status1[:maxlen-4]
status1 += "... "
}
}
fmt.Printf("\x1b[2K%s\r", status1)
}
archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
fmt.Printf("\nduration: %s, %s\n", formatDuration(d), formatRate(s.Bytes, d))
}
return archiveProgress
}
func samePaths(expected, actual []string) bool {
if expected == nil || actual == nil {
return true
@ -243,7 +286,41 @@ func filterExisting(items []string) (result []string, err error) {
return
}
func (cmd CmdBackup) readFromStdin(args []string) error {
if len(args) != 0 {
return fmt.Errorf("when reading from stdin, no additional files can be specified")
}
repo, err := cmd.global.OpenRepository()
if err != nil {
return err
}
lock, err := lockRepo(repo)
defer unlockRepo(lock)
if err != nil {
return err
}
err = repo.LoadIndex()
if err != nil {
return err
}
_, id, err := restic.ArchiveReader(repo, cmd.newArchiveStdinProgress(), os.Stdin, cmd.StdinFilename)
if err != nil {
return err
}
fmt.Printf("archived as %v\n", id.Str())
return nil
}
func (cmd CmdBackup) Execute(args []string) error {
if cmd.Stdin {
return cmd.readFromStdin(args)
}
if len(args) == 0 {
return fmt.Errorf("wrong number of parameters, Usage: %s", cmd.Usage())
}

View File

@ -0,0 +1,122 @@
package restic
import (
"encoding/json"
"io"
"restic/backend"
"restic/debug"
"restic/pack"
"restic/repository"
"time"
"github.com/restic/chunker"
)
// saveTreeJSON stores a tree in the repository.
func saveTreeJSON(repo *repository.Repository, item interface{}) (backend.ID, error) {
data, err := json.Marshal(item)
if err != nil {
return backend.ID{}, err
}
data = append(data, '\n')
// check if tree has been saved before
id := backend.Hash(data)
if repo.Index().Has(id) {
return id, nil
}
return repo.SaveJSON(pack.Tree, item)
}
// ArchiveReader reads from the reader and archives the data. Returned is the
// resulting snapshot and its ID.
func ArchiveReader(repo *repository.Repository, p *Progress, rd io.Reader, name string) (*Snapshot, backend.ID, error) {
debug.Log("ArchiveReader", "start archiving %s", name)
sn, err := NewSnapshot([]string{name})
if err != nil {
return nil, backend.ID{}, err
}
p.Start()
defer p.Done()
chnker := chunker.New(rd, repo.Config.ChunkerPolynomial)
var ids backend.IDs
var fileSize uint64
for {
chunk, err := chnker.Next(getBuf())
if err == io.EOF {
break
}
if err != nil {
return nil, backend.ID{}, err
}
id := backend.Hash(chunk.Data)
if !repo.Index().Has(id) {
_, err := repo.SaveAndEncrypt(pack.Data, chunk.Data, nil)
if err != nil {
return nil, backend.ID{}, err
}
debug.Log("ArchiveReader", "saved blob %v (%d bytes)\n", id.Str(), chunk.Length)
} else {
debug.Log("ArchiveReader", "blob %v already saved in the repo\n", id.Str())
}
freeBuf(chunk.Data)
ids = append(ids, id)
p.Report(Stat{Bytes: uint64(chunk.Length)})
fileSize += uint64(chunk.Length)
}
tree := &Tree{
Nodes: []*Node{
&Node{
Name: name,
AccessTime: time.Now(),
ModTime: time.Now(),
Type: "file",
Mode: 0644,
Size: fileSize,
UID: sn.UID,
GID: sn.GID,
User: sn.Username,
Content: ids,
},
},
}
treeID, err := saveTreeJSON(repo, tree)
if err != nil {
return nil, backend.ID{}, err
}
sn.Tree = &treeID
debug.Log("ArchiveReader", "tree saved as %v", treeID.Str())
id, err := repo.SaveJSONUnpacked(backend.Snapshot, sn)
if err != nil {
return nil, backend.ID{}, err
}
sn.id = &id
debug.Log("ArchiveReader", "snapshot saved as %v", id.Str())
err = repo.Flush()
if err != nil {
return nil, backend.ID{}, err
}
err = repo.SaveIndex()
if err != nil {
return nil, backend.ID{}, err
}
return sn, id, nil
}

View File

@ -0,0 +1,103 @@
package restic
import (
"bytes"
"io"
"math/rand"
"restic/backend"
"restic/pack"
"restic/repository"
"testing"
"github.com/restic/chunker"
)
func loadBlob(t *testing.T, repo *repository.Repository, id backend.ID, buf []byte) []byte {
buf, err := repo.LoadBlob(pack.Data, id, buf)
if err != nil {
t.Fatalf("LoadBlob(%v) returned error %v", id, err)
}
return buf
}
func checkSavedFile(t *testing.T, repo *repository.Repository, treeID backend.ID, name string, rd io.Reader) {
tree, err := LoadTree(repo, treeID)
if err != nil {
t.Fatalf("LoadTree() returned error %v", err)
}
if len(tree.Nodes) != 1 {
t.Fatalf("wrong number of nodes for tree, want %v, got %v", 1, len(tree.Nodes))
}
node := tree.Nodes[0]
if node.Name != "fakefile" {
t.Fatalf("wrong filename, want %v, got %v", "fakefile", node.Name)
}
if len(node.Content) == 0 {
t.Fatalf("node.Content has length 0")
}
// check blobs
buf := make([]byte, chunker.MaxSize)
buf2 := make([]byte, chunker.MaxSize)
for i, id := range node.Content {
buf = loadBlob(t, repo, id, buf)
buf2 = buf2[:len(buf)]
_, err = io.ReadFull(rd, buf2)
if !bytes.Equal(buf, buf2) {
t.Fatalf("blob %d (%v) is wrong", i, id.Str())
}
}
}
func TestArchiveReader(t *testing.T) {
repo, cleanup := repository.TestRepository(t)
defer cleanup()
seed := rand.Int63()
size := int64(rand.Intn(50*1024*1024) + 50*1024*1024)
t.Logf("seed is 0x%016x, size is %v", seed, size)
f := fakeFile(t, seed, size)
sn, id, err := ArchiveReader(repo, nil, f, "fakefile")
if err != nil {
t.Fatalf("ArchiveReader() returned error %v", err)
}
if id.IsNull() {
t.Fatalf("ArchiveReader() returned null ID")
}
t.Logf("snapshot saved as %v, tree is %v", id.Str(), sn.Tree.Str())
checkSavedFile(t, repo, *sn.Tree, "fakefile", fakeFile(t, seed, size))
}
func BenchmarkArchiveReader(t *testing.B) {
repo, cleanup := repository.TestRepository(t)
defer cleanup()
const size = 50 * 1024 * 1024
buf := make([]byte, size)
_, err := io.ReadFull(fakeFile(t, 23, size), buf)
if err != nil {
t.Fatal(err)
}
t.SetBytes(size)
t.ResetTimer()
for i := 0; i < t.N; i++ {
_, _, err := ArchiveReader(repo, nil, bytes.NewReader(buf), "fakefile")
if err != nil {
t.Fatal(err)
}
}
}

View File

@ -11,6 +11,7 @@ import (
"golang.org/x/net/context"
"restic"
"restic/debug"
"restic/repository"
)
@ -27,8 +28,10 @@ type dir struct {
}
func newDir(repo *repository.Repository, node *restic.Node, ownerIsRoot bool) (*dir, error) {
debug.Log("newDir", "new dir for %v (%v)", node.Name, node.Subtree.Str())
tree, err := restic.LoadTree(repo, *node.Subtree)
if err != nil {
debug.Log("newDir", " error loading tree %v: %v", node.Subtree.Str(), err)
return nil, err
}
items := make(map[string]*restic.Node)
@ -65,14 +68,17 @@ func replaceSpecialNodes(repo *repository.Repository, node *restic.Node) ([]*res
}
func newDirFromSnapshot(repo *repository.Repository, snapshot SnapshotWithId, ownerIsRoot bool) (*dir, error) {
debug.Log("newDirFromSnapshot", "new dir for snapshot %v (%v)", snapshot.ID.Str(), snapshot.Tree.Str())
tree, err := restic.LoadTree(repo, *snapshot.Tree)
if err != nil {
debug.Log("newDirFromSnapshot", " loadTree(%v) failed: %v", snapshot.ID.Str(), err)
return nil, err
}
items := make(map[string]*restic.Node)
for _, n := range tree.Nodes {
nodes, err := replaceSpecialNodes(repo, n)
if err != nil {
debug.Log("newDirFromSnapshot", " replaceSpecialNodes(%v) failed: %v", n, err)
return nil, err
}
@ -98,6 +104,7 @@ func newDirFromSnapshot(repo *repository.Repository, snapshot SnapshotWithId, ow
}
func (d *dir) Attr(ctx context.Context, a *fuse.Attr) error {
debug.Log("dir.Attr", "called")
a.Inode = d.inode
a.Mode = os.ModeDir | d.node.Mode
@ -112,6 +119,7 @@ func (d *dir) Attr(ctx context.Context, a *fuse.Attr) error {
}
func (d *dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
debug.Log("dir.ReadDirAll", "called")
ret := make([]fuse.Dirent, 0, len(d.items))
for _, node := range d.items {
@ -136,8 +144,10 @@ func (d *dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
}
func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
debug.Log("dir.Lookup", "Lookup(%v)", name)
node, ok := d.items[name]
if !ok {
debug.Log("dir.Lookup", " Lookup(%v) -> not found", name)
return nil, fuse.ENOENT
}
switch node.Type {
@ -148,6 +158,7 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
case "symlink":
return newLink(d.repo, node, d.ownerIsRoot)
default:
debug.Log("dir.Lookup", " node %v has unknown type %v", name, node.Type)
return nil, fuse.ENOENT
}
}

View File

@ -8,6 +8,7 @@ import (
"restic"
"restic/backend"
"restic/debug"
"restic/pack"
"bazil.org/fuse"
@ -47,6 +48,8 @@ var blobPool = sync.Pool{
}
func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error) {
debug.Log("newFile", "create new file for %v with %d blobs", node.Name, len(node.Content))
var bytes uint64
sizes := make([]uint, len(node.Content))
for i, id := range node.Content {
size, err := repo.LookupBlobSize(id)
@ -55,6 +58,12 @@ func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error
}
sizes[i] = size
bytes += uint64(size)
}
if bytes != node.Size {
debug.Log("newFile", "sizes do not match: node.Size %v != size %v, using real size", node.Size, bytes)
node.Size = bytes
}
return &file{
@ -67,6 +76,7 @@ func newFile(repo BlobLoader, node *restic.Node, ownerIsRoot bool) (*file, error
}
func (f *file) Attr(ctx context.Context, a *fuse.Attr) error {
debug.Log("file.Attr", "Attr(%v)", f.node.Name)
a.Inode = f.node.Inode
a.Mode = f.node.Mode
a.Size = f.node.Size
@ -84,6 +94,7 @@ func (f *file) Attr(ctx context.Context, a *fuse.Attr) error {
}
func (f *file) getBlobAt(i int) (blob []byte, err error) {
debug.Log("file.getBlobAt", "getBlobAt(%v, %v)", f.node.Name, i)
if f.blobs[i] != nil {
return f.blobs[i], nil
}
@ -100,6 +111,7 @@ func (f *file) getBlobAt(i int) (blob []byte, err error) {
blob, err = f.repo.LoadBlob(pack.Data, f.node.Content[i], buf)
if err != nil {
debug.Log("file.getBlobAt", "LoadBlob(%v, %v) failed: %v", f.node.Name, f.node.Content[i], err)
return nil, err
}
f.blobs[i] = blob
@ -108,6 +120,7 @@ func (f *file) getBlobAt(i int) (blob []byte, err error) {
}
func (f *file) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
debug.Log("file.Read", "Read(%v), file size %v", req.Size, f.node.Size)
offset := req.Offset
// Skip blobs before the offset

View File

@ -13,6 +13,7 @@ import (
"restic"
"restic/backend"
"restic/debug"
"restic/repository"
"golang.org/x/net/context"
@ -39,6 +40,7 @@ type SnapshotsDir struct {
}
func NewSnapshotsDir(repo *repository.Repository, ownerIsRoot bool) *SnapshotsDir {
debug.Log("NewSnapshotsDir", "fuse mount initiated")
return &SnapshotsDir{
repo: repo,
knownSnapshots: make(map[string]SnapshotWithId),
@ -54,10 +56,12 @@ func (sn *SnapshotsDir) Attr(ctx context.Context, attr *fuse.Attr) error {
attr.Uid = uint32(os.Getuid())
attr.Gid = uint32(os.Getgid())
}
debug.Log("SnapshotsDir.Attr", "attr is %v", attr)
return nil
}
func (sn *SnapshotsDir) updateCache(ctx context.Context) error {
debug.Log("SnapshotsDir.updateCache", "called")
sn.Lock()
defer sn.Unlock()
@ -75,10 +79,12 @@ func (sn *SnapshotsDir) get(name string) (snapshot SnapshotWithId, ok bool) {
sn.RLock()
snapshot, ok = sn.knownSnapshots[name]
sn.RUnlock()
debug.Log("SnapshotsDir.get", "get(%s) -> %v %v", name, snapshot, ok)
return snapshot, ok
}
func (sn *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
debug.Log("SnapshotsDir.ReadDirAll", "called")
err := sn.updateCache(ctx)
if err != nil {
return nil, err
@ -96,21 +102,25 @@ func (sn *SnapshotsDir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
})
}
debug.Log("SnapshotsDir.ReadDirAll", " -> %d entries", len(ret))
return ret, nil
}
func (sn *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error) {
debug.Log("SnapshotsDir.updateCache", "Lookup(%s)", name)
snapshot, ok := sn.get(name)
if !ok {
// We don't know about it, update the cache
err := sn.updateCache(ctx)
if err != nil {
debug.Log("SnapshotsDir.updateCache", " Lookup(%s) -> err %v", name, err)
return nil, err
}
snapshot, ok = sn.get(name)
if !ok {
// We still don't know about it, this time it really doesn't exist
debug.Log("SnapshotsDir.updateCache", " Lookup(%s) -> not found", name)
return nil, fuse.ENOENT
}
}