rdelaage
/
restic
mirror of https://github.com/restic/restic.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #194 from restic/fix-orphaned-check 

Fix fsck orphaned check, add debug command 'dump', cleanup
This commit is contained in:
Alexander Neumann 2015-05-28 21:40:23 +02:00
parent 219f857c67 0314d2b0c3
commit 8eec59ebb2
16 changed files with 296 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
archiver.go
View File

@ -254,7 +254,7 @@ func (arch *Archiver) fileWorker(wg *sync.WaitGroup, p *Progress, done <-chan st
// check if all content is still available in the repository
contentMissing := false
for _, blob := range oldNode.blobs {
if ok, err := arch.repo.Test(backend.Data, blob.Storage.String()); !ok || err != nil {
if ok, err := arch.repo.Backend().Test(backend.Data, blob.Storage.String()); !ok || err != nil {
debug.Log("Archiver.fileWorker", " %v not using old data, %v (%v) is missing", e.Path(), blob.ID.Str(), blob.Storage.Str())
contentMissing = true
break

12
backend/generic.go
View File

@ -56,18 +56,6 @@ func Find(be Lister, t Type, prefix string) (string, error) {
return "", ErrNoIDPrefixFound
}
// FindSnapshot takes a string and tries to find a snapshot whose ID matches
// the string as closely as possible.
func FindSnapshot(be Lister, s string) (string, error) {
// find snapshot id with prefix
name, err := Find(be, Snapshot, s)
if err != nil {
return "", err
}
return name, nil
}
// PrefixLength returns the number of bytes required so that all prefixes of
// all names of type t are unique.
func PrefixLength(be Lister, t Type) (int, error) {

2
cache.go
View File

@ -115,7 +115,7 @@ func (c *Cache) Clear(repo *repository.Repository) error {
for _, entry := range list {
debug.Log("Cache.Clear", "found entry %v", entry)
if ok, err := repo.Test(backend.Snapshot, entry.ID.String()); !ok || err != nil {
if ok, err := repo.Backend().Test(backend.Snapshot, entry.ID.String()); !ok || err != nil {
debug.Log("Cache.Clear", "snapshot %v doesn't exist any more, removing %v", entry.ID, entry)
err = c.purge(backend.Snapshot, entry.Subtype, entry.ID)

29
cmd/restic/cmd_backup.go
View File

@ -185,28 +185,20 @@ func (cmd CmdBackup) Execute(args []string) error {
return err
}
var (
parentSnapshot string
parentSnapshotID backend.ID
)
var parentSnapshotID backend.ID
// Force using a parent
if !cmd.Force && cmd.Parent != "" {
parentSnapshot, err = s.FindSnapshot(cmd.Parent)
parentSnapshotID, err = restic.FindSnapshot(s, cmd.Parent)
if err != nil {
return fmt.Errorf("invalid id %q: %v", cmd.Parent, err)
}
parentSnapshotID, err = backend.ParseID(parentSnapshot)
if err != nil {
return fmt.Errorf("invalid parent snapshot id %v", parentSnapshot)
}
fmt.Printf("found parent snapshot %v\n", parentSnapshotID)
}
// Find last snapshot to set it as parent, if not already set
if !cmd.Force && parentSnapshot == "" {
if !cmd.Force && parentSnapshotID == nil {
samePaths := func(expected, actual []string) bool {
if len(expected) != len(actual) {
return false
@ -221,24 +213,19 @@ func (cmd CmdBackup) Execute(args []string) error {
var latest time.Time
for snapshotIDString := range s.List(backend.Snapshot, make(chan struct{})) {
snapshotID, err := backend.ParseID(snapshotIDString)
if err != nil {
return fmt.Errorf("Error with the listing of snapshots inputting invalid backend ids: %v", err)
}
snapshot, err := restic.LoadSnapshot(s, snapshotID)
for id := range s.List(backend.Snapshot, make(chan struct{})) {
snapshot, err := restic.LoadSnapshot(s, id)
if err != nil {
return fmt.Errorf("Error listing snapshot: %v", err)
}
if snapshot.Time.After(latest) && samePaths(snapshot.Paths, target) {
latest = snapshot.Time
parentSnapshotID = snapshotID
parentSnapshot = snapshotIDString
parentSnapshotID = id
}
}
if parentSnapshot != "" {
if parentSnapshotID != nil {
fmt.Printf("using parent snapshot %v\n", parentSnapshotID)
}
}

7
cmd/restic/cmd_cat.go
View File

@ -53,12 +53,7 @@ func (cmd CmdCat) Execute(args []string) error {
}
// find snapshot id with prefix
name, err := s.FindSnapshot(args[1])
if err != nil {
return err
}
id, err = backend.ParseID(name)
id, err = restic.FindSnapshot(s, args[1])
if err != nil {
return err
}

149
cmd/restic/cmd_dump.go Normal file
View File

@ -0,0 +1,149 @@
// +build debug
package main
import (
"encoding/json"
"fmt"
"io"
"os"
"github.com/juju/errors"
"github.com/restic/restic"
"github.com/restic/restic/backend"
"github.com/restic/restic/pack"
"github.com/restic/restic/repository"
)
type CmdDump struct{}
func init() {
_, err := parser.AddCommand("dump",
"dump data structures",
"The dump command dumps data structures from a repository as JSON documents",
&CmdDump{})
if err != nil {
panic(err)
}
}
func dumpIndex(r *repository.Repository, wr io.Writer) error {
fmt.Fprintln(wr, "foo")
return nil
}
func (cmd CmdDump) Usage() string {
return "[index|snapshots|trees|all]"
}
func prettyPrintJSON(wr io.Writer, item interface{}) error {
buf, err := json.MarshalIndent(item, "", " ")
if err != nil {
return err
}
_, err = wr.Write(append(buf, '\n'))
return err
}
func printSnapshots(repo *repository.Repository, wr io.Writer) error {
done := make(chan struct{})
defer close(done)
for id := range repo.List(backend.Snapshot, done) {
snapshot, err := restic.LoadSnapshot(repo, id)
if err != nil {
fmt.Fprintf(os.Stderr, "LoadSnapshot(%v): %v", id.Str(), err)
continue
}
fmt.Fprintf(wr, "snapshot_id: %v\n", id)
err = prettyPrintJSON(wr, snapshot)
if err != nil {
return err
}
}
return nil
}
func printTrees(repo *repository.Repository, wr io.Writer) error {
done := make(chan struct{})
defer close(done)
trees := []backend.ID{}
for blob := range repo.Index().Each(done) {
if blob.Type != pack.Tree {
continue
}
trees = append(trees, blob.ID)
}
for _, id := range trees {
tree, err := restic.LoadTree(repo, id)
if err != nil {
fmt.Fprintf(os.Stderr, "LoadTree(%v): %v", id.Str(), err)
continue
}
fmt.Fprintf(wr, "tree_id: %v\n", id)
prettyPrintJSON(wr, tree)
}
return nil
}
func (cmd CmdDump) Execute(args []string) error {
if len(args) != 1 {
return fmt.Errorf("type not specified, Usage: %s", cmd.Usage())
}
repo, err := OpenRepo()
if err != nil {
return err
}
err = repo.LoadIndex()
if err != nil {
return err
}
tpe := args[0]
switch tpe {
case "index":
return repo.Index().Dump(os.Stdout)
case "snapshots":
return printSnapshots(repo, os.Stdout)
case "trees":
return printTrees(repo, os.Stdout)
case "all":
fmt.Printf("snapshots:\n")
err := printSnapshots(repo, os.Stdout)
if err != nil {
return err
}
fmt.Printf("\ntrees:\n")
err = printTrees(repo, os.Stdout)
if err != nil {
return err
}
fmt.Printf("\nindex:\n")
err = repo.Index().Dump(os.Stdout)
if err != nil {
return err
}
return nil
default:
return errors.Errorf("no such type %q", tpe)
}
}

11
cmd/restic/cmd_find.go
View File

@ -105,13 +105,8 @@ func (c CmdFind) findInTree(repo *repository.Repository, id backend.ID, path str
return results, nil
}
func (c CmdFind) findInSnapshot(repo *repository.Repository, name string) error {
debug.Log("restic.find", "searching in snapshot %s\n for entries within [%s %s]", name, c.oldest, c.newest)
id, err := backend.ParseID(name)
if err != nil {
return err
}
func (c CmdFind) findInSnapshot(repo *repository.Repository, id backend.ID) error {
debug.Log("restic.find", "searching in snapshot %s\n for entries within [%s %s]", id.Str(), c.oldest, c.newest)
sn, err := restic.LoadSnapshot(repo, id)
if err != nil {
@ -169,7 +164,7 @@ func (c CmdFind) Execute(args []string) error {
c.pattern = args[0]
if c.Snapshot != "" {
snapshotID, err := backend.FindSnapshot(s, c.Snapshot)
snapshotID, err := restic.FindSnapshot(s, c.Snapshot)
if err != nil {
return fmt.Errorf("invalid id %q: %v", args[1], err)
}

28
cmd/restic/cmd_fsck.go
View File

@ -57,8 +57,8 @@ func fsckFile(opts CmdFsck, repo *repository.Repository, IDs []backend.ID) (uint
return 0, err
}
} else {
// test if data blob is there
ok, err := repo.Test(backend.Data, packID.String())
// test if pack for data blob is there
ok, err := repo.Backend().Test(backend.Data, packID.String())
if err != nil {
return 0, err
}
@ -70,6 +70,7 @@ func fsckFile(opts CmdFsck, repo *repository.Repository, IDs []backend.ID) (uint
// if orphan check is active, record storage id
if opts.o_data != nil {
debug.Log("restic.fsck", " recording blob %v as used\n", id)
opts.o_data.Insert(id)
}
}
@ -198,16 +199,11 @@ func (cmd CmdFsck) Execute(args []string) error {
}
if cmd.Snapshot != "" {
name, err := s.FindSnapshot(cmd.Snapshot)
id, err := restic.FindSnapshot(s, cmd.Snapshot)
if err != nil {
return fmt.Errorf("invalid id %q: %v", cmd.Snapshot, err)
}
id, err := backend.ParseID(name)
if err != nil {
fmt.Fprintf(os.Stderr, "invalid snapshot id %v\n", name)
}
err = fsckSnapshot(cmd, s, id)
if err != nil {
fmt.Fprintf(os.Stderr, "check for snapshot %v failed\n", id)
@ -225,13 +221,7 @@ func (cmd CmdFsck) Execute(args []string) error {
defer close(done)
var firstErr error
for name := range s.List(backend.Snapshot, done) {
id, err := backend.ParseID(name)
if err != nil {
fmt.Fprintf(os.Stderr, "invalid snapshot id %v\n", name)
continue
}
for id := range s.List(backend.Snapshot, done) {
err = fsckSnapshot(cmd, s, id)
if err != nil {
fmt.Fprintf(os.Stderr, "check for snapshot %v failed\n", id)
@ -246,14 +236,16 @@ func (cmd CmdFsck) Execute(args []string) error {
debug.Log("restic.fsck", "starting orphaned check\n")
cnt := make(map[pack.BlobType]*backend.IDSet)
cnt[pack.Data] = backend.NewIDSet()
cnt[pack.Tree] = backend.NewIDSet()
cnt[pack.Data] = cmd.o_data
cnt[pack.Tree] = cmd.o_trees
for blob := range s.Index().Each(done) {
fmt.Println(blob.ID)
debug.Log("restic.fsck", "checking %v blob %v\n", blob.Type, blob.ID)
err = cnt[blob.Type].Find(blob.ID)
if err != nil {
debug.Log("restic.fsck", " blob %v is orphaned\n", blob.ID)
if !cmd.RemoveOrphaned {
fmt.Printf("orphaned %v blob %v\n", blob.Type, blob.ID)
continue

14
cmd/restic/cmd_key.go
View File

@ -34,20 +34,20 @@ func listKeys(s *repository.Repository) error {
done := make(chan struct{})
defer close(done)
for name := range s.List(backend.Key, done) {
k, err := repository.LoadKey(s, name)
for id := range s.List(backend.Key, done) {
k, err := repository.LoadKey(s, id.String())
if err != nil {
fmt.Fprintf(os.Stderr, "LoadKey() failed: %v\n", err)
continue
}
var current string
if name == s.KeyName() {
if id.String() == s.KeyName() {
current = "*"
} else {
current = " "
}
tab.Rows = append(tab.Rows, []interface{}{current, name[:plen],
tab.Rows = append(tab.Rows, []interface{}{current, id.String()[:plen],
k.Username, k.Hostname, k.Created.Format(TimeFormat)})
}
@ -79,7 +79,7 @@ func deleteKey(repo *repository.Repository, name string) error {
return errors.New("refusing to remove key currently used to access repository")
}
err := repo.Remove(backend.Key, name)
err := repo.Backend().Remove(backend.Key, name)
if err != nil {
return err
}
@ -103,7 +103,7 @@ func changePassword(s *repository.Repository) error {
}
// remove old key
err = s.Remove(backend.Key, s.KeyName())
err = s.Backend().Remove(backend.Key, s.KeyName())
if err != nil {
return err
}
@ -133,7 +133,7 @@ func (cmd CmdKey) Execute(args []string) error {
case "add":
return addKey(s)
case "rm":
id, err := backend.Find(s, backend.Key, args[1])
id, err := backend.Find(s.Backend(), backend.Key, args[1])
if err != nil {
return err
}

7
cmd/restic/cmd_ls.go
View File

@ -77,12 +77,7 @@ func (cmd CmdLs) Execute(args []string) error {
return err
}
name, err := backend.FindSnapshot(s, args[0])
if err != nil {
return err
}
id, err := backend.ParseID(name)
id, err := restic.FindSnapshot(s, args[0])
if err != nil {
return err
}

8
cmd/restic/cmd_restore.go
View File

@ -6,7 +6,6 @@ import (
"path/filepath"
"github.com/restic/restic"
"github.com/restic/restic/backend"
)
type CmdRestore struct{}
@ -40,12 +39,7 @@ func (cmd CmdRestore) Execute(args []string) error {
return err
}
name, err := backend.FindSnapshot(s, args[0])
if err != nil {
errx(1, "invalid id %q: %v", args[0], err)
}
id, err := backend.ParseID(name)
id, err := restic.FindSnapshot(s, args[0])
if err != nil {
errx(1, "invalid id %q: %v", args[0], err)
}

8
cmd/restic/cmd_snapshots.go
View File

@ -105,13 +105,7 @@ func (cmd CmdSnapshots) Execute(args []string) error {
defer close(done)
list := []*restic.Snapshot{}
for name := range s.List(backend.Snapshot, done) {
id, err := backend.ParseID(name)
if err != nil {
fmt.Fprintf(os.Stderr, "error parsing id: %v", name)
continue
}
for id := range s.List(backend.Snapshot, done) {
sn, err := restic.LoadSnapshot(s, id)
if err != nil {
fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err)

71
repository/index.go
View File

@ -173,28 +173,24 @@ type blobJSON struct {
Length uint `json:"length"`
}
// Encode writes the JSON serialization of the index to the writer w. This
// serialization only contains new blobs added via idx.Store(), not old ones
// introduced via DecodeIndex().
func (idx *Index) Encode(w io.Writer) error {
debug.Log("Index.Encode", "encoding index")
idx.m.Lock()
defer idx.m.Unlock()
// generatePackList returns a list of packs containing only the index entries
// that selsectFn returned true for. If selectFn is nil, the list contains all
// blobs in the index.
func (idx *Index) generatePackList(selectFn func(indexEntry) bool) ([]*packJSON, error) {
list := []*packJSON{}
packs := make(map[string]*packJSON)
for id, blob := range idx.pack {
if blob.old {
if selectFn != nil && !selectFn(blob) {
continue
}
debug.Log("Index.Encode", "handle blob %q", id[:8])
debug.Log("Index.generatePackList", "handle blob %q", id[:8])
if blob.packID == nil {
debug.Log("Index.Encode", "blob %q has no packID! (type %v, offset %v, length %v)",
debug.Log("Index.generatePackList", "blob %q has no packID! (type %v, offset %v, length %v)",
id[:8], blob.tpe, blob.offset, blob.length)
return fmt.Errorf("unable to serialize index: pack for blob %v hasn't been written yet", id)
return nil, fmt.Errorf("unable to serialize index: pack for blob %v hasn't been written yet", id)
}
// see if pack is already in map
@ -217,12 +213,63 @@ func (idx *Index) Encode(w io.Writer) error {
})
}
debug.Log("Index.generatePackList", "done")
return list, nil
}
// encode writes the JSON serialization of the index filtered by selectFn to enc.
func (idx *Index) encode(w io.Writer, selectFn func(indexEntry) bool) error {
list, err := idx.generatePackList(func(entry indexEntry) bool {
return !entry.old
})
if err != nil {
return err
}
debug.Log("Index.Encode", "done")
enc := json.NewEncoder(w)
return enc.Encode(list)
}
// Encode writes the JSON serialization of the index to the writer w. This
// serialization only contains new blobs added via idx.Store(), not old ones
// introduced via DecodeIndex().
func (idx *Index) Encode(w io.Writer) error {
debug.Log("Index.Encode", "encoding index")
idx.m.Lock()
defer idx.m.Unlock()
return idx.encode(w, func(e indexEntry) bool { return !e.old })
}
// Dump writes the pretty-printed JSON representation of the index to w.
func (idx *Index) Dump(w io.Writer) error {
debug.Log("Index.Dump", "dumping index")
idx.m.Lock()
defer idx.m.Unlock()
list, err := idx.generatePackList(nil)
if err != nil {
return err
}
buf, err := json.MarshalIndent(list, "", " ")
if err != nil {
return err
}
_, err = w.Write(append(buf, '\n'))
if err != nil {
return err
}
debug.Log("Index.Dump", "done")
return nil
}
// DecodeIndex loads and unserializes an index from rd.
func DecodeIndex(rd io.Reader) (*Index, error) {
debug.Log("Index.DecodeIndex", "Start decoding index")

2
repository/key.go
View File

@ -98,7 +98,7 @@ func SearchKey(s *Repository, password string) (*Key, error) {
// try all keys in repo
done := make(chan struct{})
defer close(done)
for name := range s.List(backend.Key, done) {
for name := range s.Backend().List(backend.Key, done) {
key, err := OpenKey(s, name, password)
if err != nil {
continue

65
repository/repository.go
View File

@ -52,12 +52,6 @@ func (s *Repository) Find(t backend.Type, prefix string) (string, error) {
return backend.Find(s.be, t, prefix)
}
// FindSnapshot takes a string and tries to find a snapshot whose ID matches
// the string as closely as possible.
func (s *Repository) FindSnapshot(name string) (string, error) {
return backend.FindSnapshot(s.be, name)
}
// PrefixLength returns the number of bytes required so that all prefixes of
// all IDs of type t are unique.
func (s *Repository) PrefixLength(t backend.Type) (int, error) {
@ -586,7 +580,7 @@ func (s *Repository) SearchKey(password string) error {
// Init creates a new master key with the supplied password and initializes the
// repository config.
func (s *Repository) Init(password string) error {
has, err := s.Test(backend.Config, "")
has, err := s.be.Test(backend.Config, "")
if err != nil {
return err
}
@ -637,26 +631,49 @@ func (s *Repository) Count(t backend.Type) (n uint) {
return
}
// Proxy methods to backend
func (s *Repository) list(t backend.Type, done <-chan struct{}, out chan<- backend.ID) {
defer close(out)
in := s.be.List(t, done)
func (s *Repository) Get(t backend.Type, name string) (io.ReadCloser, error) {
return s.be.Get(t, name)
var (
// disable sending on the outCh until we received a job
outCh chan<- backend.ID
// enable receiving from in
inCh = in
id backend.ID
err error
)
for {
select {
case <-done:
return
case strID, ok := <-inCh:
if !ok {
// input channel closed, we're done
return
}
id, err = backend.ParseID(strID)
if err != nil {
// ignore invalid IDs
continue
}
inCh = nil
outCh = out
case outCh <- id:
outCh = nil
inCh = in
}
}
}
func (s *Repository) List(t backend.Type, done <-chan struct{}) <-chan string {
return s.be.List(t, done)
}
func (s *Repository) List(t backend.Type, done <-chan struct{}) <-chan backend.ID {
outCh := make(chan backend.ID)
func (s *Repository) Test(t backend.Type, name string) (bool, error) {
return s.be.Test(t, name)
}
go s.list(t, done, outCh)
func (s *Repository) Remove(t backend.Type, name string) error {
return s.be.Remove(t, name)
}
func (s *Repository) Close() error {
return s.be.Close()
return outCh
}
func (s *Repository) Delete() error {
@ -667,6 +684,6 @@ func (s *Repository) Delete() error {
return errors.New("Delete() called for backend that does not implement this method")
}
func (s *Repository) Location() string {
return s.be.Location()
func (s *Repository) Close() error {
return s.be.Close()
}

12
snapshot.go
View File

@ -89,3 +89,15 @@ func (sn *Snapshot) fillUserInfo() error {
return nil
}
// FindSnapshot takes a string and tries to find a snapshot whose ID matches
// the string as closely as possible.
func FindSnapshot(repo *repository.Repository, s string) (backend.ID, error) {
// find snapshot id with prefix
name, err := backend.Find(repo.Backend(), backend.Snapshot, s)
if err != nil {
return nil, err
}
return backend.ParseID(name)
}