From 189a33730a882c6de6ffb6cb7fec4e521bcc6d7c Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Fri, 26 Jun 2015 22:12:04 +0200 Subject: [PATCH 01/12] tests: Standardize use of SetupRepo/Teardown --- archiver_test.go | 36 +++++++++++++++++------------------ cache_test.go | 4 ++-- repository/repository_test.go | 32 +++++++++++++++---------------- snapshot_test.go | 3 --- test/backend.go | 31 +++++++++++++++++++++++------- tree_test.go | 4 ++-- walk_test.go | 6 +++--- 7 files changed, 65 insertions(+), 51 deletions(-) diff --git a/archiver_test.go b/archiver_test.go index 9aa9bffb4..5f8ee6bd0 100644 --- a/archiver_test.go +++ b/archiver_test.go @@ -48,12 +48,12 @@ func benchmarkChunkEncrypt(b testing.TB, buf, buf2 []byte, rd Rdr, key *crypto.K } func BenchmarkChunkEncrypt(b *testing.B) { + repo := SetupRepo() + defer TeardownRepo(repo) + data := Random(23, 10<<20) // 10MiB rd := bytes.NewReader(data) - s := SetupRepo(b) - defer TeardownRepo(b, s) - buf := make([]byte, chunker.MaxSize) buf2 := make([]byte, chunker.MaxSize) @@ -61,7 +61,7 @@ func BenchmarkChunkEncrypt(b *testing.B) { b.SetBytes(int64(len(data))) for i := 0; i < b.N; i++ { - benchmarkChunkEncrypt(b, buf, buf2, rd, s.Key()) + benchmarkChunkEncrypt(b, buf, buf2, rd, repo.Key()) } } @@ -82,8 +82,8 @@ func benchmarkChunkEncryptP(b *testing.PB, buf []byte, rd Rdr, key *crypto.Key) } func BenchmarkChunkEncryptParallel(b *testing.B) { - s := SetupRepo(b) - defer TeardownRepo(b, s) + repo := SetupRepo() + defer TeardownRepo(repo) data := Random(23, 10<<20) // 10MiB @@ -95,14 +95,14 @@ func BenchmarkChunkEncryptParallel(b *testing.B) { b.RunParallel(func(pb *testing.PB) { for pb.Next() { rd := bytes.NewReader(data) - benchmarkChunkEncryptP(pb, buf, rd, s.Key()) + benchmarkChunkEncryptP(pb, buf, rd, repo.Key()) } }) } func archiveDirectory(b testing.TB) { - repo := SetupRepo(b) - defer TeardownRepo(b, repo) + repo := SetupRepo() + defer TeardownRepo(repo) arch := restic.NewArchiver(repo) @@ -131,13 +131,13 @@ func BenchmarkArchiveDirectory(b *testing.B) { } func archiveWithDedup(t testing.TB) { + repo := SetupRepo() + defer TeardownRepo(repo) + if *benchArchiveDirectory == "" { t.Skip("benchdir not set, skipping TestArchiverDedup") } - repo := SetupRepo(t) - defer TeardownRepo(t, repo) - var cnt struct { before, after, after2 struct { packs, dataBlobs, treeBlobs uint @@ -195,15 +195,15 @@ func TestArchiveDedup(t *testing.T) { } func BenchmarkLoadTree(t *testing.B) { + repo := SetupRepo() + defer TeardownRepo(repo) + if *benchArchiveDirectory == "" { t.Skip("benchdir not set, skipping TestArchiverDedup") } - s := SetupRepo(t) - defer TeardownRepo(t, s) - // archive a few files - arch := restic.NewArchiver(s) + arch := restic.NewArchiver(repo) sn, _, err := arch.Snapshot(nil, []string{*benchArchiveDirectory}, nil) OK(t, err) t.Logf("archived snapshot %v", sn.ID()) @@ -211,7 +211,7 @@ func BenchmarkLoadTree(t *testing.B) { list := make([]backend.ID, 0, 10) done := make(chan struct{}) - for blob := range s.Index().Each(done) { + for blob := range repo.Index().Each(done) { if blob.Type != pack.Tree { continue } @@ -228,7 +228,7 @@ func BenchmarkLoadTree(t *testing.B) { for i := 0; i < t.N; i++ { for _, id := range list { - _, err := restic.LoadTree(s, id) + _, err := restic.LoadTree(repo, id) OK(t, err) } } diff --git a/cache_test.go b/cache_test.go index 6f342dc39..9e39e86d9 100644 --- a/cache_test.go +++ b/cache_test.go @@ -8,8 +8,8 @@ import ( ) func TestCache(t *testing.T) { - repo := SetupRepo(t) - defer TeardownRepo(t, repo) + repo := SetupRepo() + defer TeardownRepo(repo) _, err := restic.NewCache(repo, "") OK(t, err) diff --git a/repository/repository_test.go b/repository/repository_test.go index fe9f5c9d4..2ccb0b284 100644 --- a/repository/repository_test.go +++ b/repository/repository_test.go @@ -28,8 +28,8 @@ var repoTests = []testJSONStruct{ } func TestSaveJSON(t *testing.T) { - repo := SetupRepo(t) - defer TeardownRepo(t, repo) + repo := SetupRepo() + defer TeardownRepo(repo) for _, obj := range repoTests { data, err := json.Marshal(obj) @@ -47,8 +47,8 @@ func TestSaveJSON(t *testing.T) { } func BenchmarkSaveJSON(t *testing.B) { - repo := SetupRepo(t) - defer TeardownRepo(t, repo) + repo := SetupRepo() + defer TeardownRepo(repo) obj := repoTests[0] @@ -72,8 +72,8 @@ func BenchmarkSaveJSON(t *testing.B) { var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20} func TestSave(t *testing.T) { - repo := SetupRepo(t) - defer TeardownRepo(t, repo) + repo := SetupRepo() + defer TeardownRepo(repo) for _, size := range testSizes { data := make([]byte, size) @@ -104,8 +104,8 @@ func TestSave(t *testing.T) { } func TestSaveFrom(t *testing.T) { - repo := SetupRepo(t) - defer TeardownRepo(t, repo) + repo := SetupRepo() + defer TeardownRepo(repo) for _, size := range testSizes { data := make([]byte, size) @@ -134,8 +134,8 @@ func TestSaveFrom(t *testing.T) { } func BenchmarkSaveFrom(t *testing.B) { - repo := SetupRepo(t) - defer TeardownRepo(t, repo) + repo := SetupRepo() + defer TeardownRepo(repo) size := 4 << 20 // 4MiB @@ -156,13 +156,13 @@ func BenchmarkSaveFrom(t *testing.B) { } func TestLoadJSONPack(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + if *benchTestDir == "" { t.Skip("benchdir not set, skipping") } - repo := SetupRepo(t) - defer TeardownRepo(t, repo) - // archive a few files sn := SnapshotDir(t, repo, *benchTestDir, nil) OK(t, repo.Flush()) @@ -173,13 +173,13 @@ func TestLoadJSONPack(t *testing.T) { } func TestLoadJSONUnpacked(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + if *benchTestDir == "" { t.Skip("benchdir not set, skipping") } - repo := SetupRepo(t) - defer TeardownRepo(t, repo) - // archive a snapshot sn := restic.Snapshot{} sn.Hostname = "foobar" diff --git a/snapshot_test.go b/snapshot_test.go index 00bb7a038..649e9a1f4 100644 --- a/snapshot_test.go +++ b/snapshot_test.go @@ -8,9 +8,6 @@ import ( ) func TestNewSnapshot(t *testing.T) { - s := SetupRepo(t) - defer TeardownRepo(t, s) - paths := []string{"/home/foobar"} _, err := restic.NewSnapshot(paths) diff --git a/test/backend.go b/test/backend.go index bcf874c0f..60aaa1249 100644 --- a/test/backend.go +++ b/test/backend.go @@ -45,27 +45,38 @@ func getBoolVar(name string, defaultValue bool) bool { return defaultValue } -func SetupRepo(t testing.TB) *repository.Repository { +func SetupRepo() *repository.Repository { tempdir, err := ioutil.TempDir(TestTempDir, "restic-test-") - OK(t, err) + if err != nil { + panic(err) + } // create repository below temp dir b, err := local.Create(filepath.Join(tempdir, "repo")) - OK(t, err) + if err != nil { + panic(err) + } repo := repository.New(b) - OK(t, repo.Init(TestPassword)) + err = repo.Init(TestPassword) + if err != nil { + panic(err) + } + return repo } -func TeardownRepo(t testing.TB, repo *repository.Repository) { +func TeardownRepo(repo *repository.Repository) { if !TestCleanup { l := repo.Backend().(*local.Local) - t.Logf("leaving local backend at %s\n", l.Location()) + fmt.Printf("leaving local backend at %s\n", l.Location()) return } - OK(t, repo.Delete()) + err := repo.Delete() + if err != nil { + panic(err) + } } func SnapshotDir(t testing.TB, repo *repository.Repository, path string, parent backend.ID) *restic.Snapshot { @@ -74,3 +85,9 @@ func SnapshotDir(t testing.TB, repo *repository.Repository, path string, parent OK(t, err) return sn } + +func WithRepo(t testing.TB, f func(*repository.Repository)) { + repo := SetupRepo() + f(repo) + TeardownRepo(repo) +} diff --git a/tree_test.go b/tree_test.go index b46f01b0a..2d9cb7b7e 100644 --- a/tree_test.go +++ b/tree_test.go @@ -93,8 +93,8 @@ func TestNodeComparison(t *testing.T) { } func TestLoadTree(t *testing.T) { - repo := SetupRepo(t) - defer TeardownRepo(t, repo) + repo := SetupRepo() + defer TeardownRepo(repo) // save tree tree := restic.NewTree() diff --git a/walk_test.go b/walk_test.go index 384e605df..15ff87f01 100644 --- a/walk_test.go +++ b/walk_test.go @@ -13,12 +13,12 @@ import ( var testWalkDirectory = flag.String("test.walkdir", ".", "test walking a directory (globbing pattern, default: .)") func TestWalkTree(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + dirs, err := filepath.Glob(*testWalkDirectory) OK(t, err) - repo := SetupRepo(t) - defer TeardownRepo(t, repo) - // archive a few files arch := restic.NewArchiver(repo) sn, _, err := arch.Snapshot(nil, dirs, nil) From 26e4d2e0198a0ec759eff7aec817609bed795598 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 28 Jun 2015 13:15:35 +0200 Subject: [PATCH 02/12] tests: Remove more flags --- archiver_test.go | 20 +++++++-------- cache_test.go | 2 +- chunker/chunker_test.go | 29 ++------------------- crypto/crypto_test.go | 13 +++++----- pipe/pipe_test.go | 48 +++++++++++++++++------------------ repository/repository_test.go | 9 +++---- test/backend.go | 13 +++++----- walk_test.go | 5 +--- 8 files changed, 52 insertions(+), 87 deletions(-) diff --git a/archiver_test.go b/archiver_test.go index 5f8ee6bd0..c639be6ab 100644 --- a/archiver_test.go +++ b/archiver_test.go @@ -3,7 +3,6 @@ package restic_test import ( "bytes" "crypto/sha256" - "flag" "io" "testing" @@ -15,7 +14,6 @@ import ( . "github.com/restic/restic/test" ) -var benchArchiveDirectory = flag.String("test.benchdir", ".", "benchmark archiving a real directory (default: .)") var testPol = chunker.Pol(0x3DA3358B4DC173) type Rdr interface { @@ -106,14 +104,14 @@ func archiveDirectory(b testing.TB) { arch := restic.NewArchiver(repo) - _, id, err := arch.Snapshot(nil, []string{*benchArchiveDirectory}, nil) + _, id, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil) OK(b, err) b.Logf("snapshot archived as %v", id) } func TestArchiveDirectory(t *testing.T) { - if *benchArchiveDirectory == "" { + if BenchArchiveDirectory == "" { t.Skip("benchdir not set, skipping TestArchiveDirectory") } @@ -121,7 +119,7 @@ func TestArchiveDirectory(t *testing.T) { } func BenchmarkArchiveDirectory(b *testing.B) { - if *benchArchiveDirectory == "" { + if BenchArchiveDirectory == "" { b.Skip("benchdir not set, skipping BenchmarkArchiveDirectory") } @@ -134,7 +132,7 @@ func archiveWithDedup(t testing.TB) { repo := SetupRepo() defer TeardownRepo(repo) - if *benchArchiveDirectory == "" { + if BenchArchiveDirectory == "" { t.Skip("benchdir not set, skipping TestArchiverDedup") } @@ -145,7 +143,7 @@ func archiveWithDedup(t testing.TB) { } // archive a few files - sn := SnapshotDir(t, repo, *benchArchiveDirectory, nil) + sn := SnapshotDir(t, repo, BenchArchiveDirectory, nil) t.Logf("archived snapshot %v", sn.ID().Str()) // get archive stats @@ -156,7 +154,7 @@ func archiveWithDedup(t testing.TB) { cnt.before.packs, cnt.before.dataBlobs, cnt.before.treeBlobs) // archive the same files again, without parent snapshot - sn2 := SnapshotDir(t, repo, *benchArchiveDirectory, nil) + sn2 := SnapshotDir(t, repo, BenchArchiveDirectory, nil) t.Logf("archived snapshot %v", sn2.ID().Str()) // get archive stats again @@ -173,7 +171,7 @@ func archiveWithDedup(t testing.TB) { } // archive the same files again, with a parent snapshot - sn3 := SnapshotDir(t, repo, *benchArchiveDirectory, sn2.ID()) + sn3 := SnapshotDir(t, repo, BenchArchiveDirectory, sn2.ID()) t.Logf("archived snapshot %v, parent %v", sn3.ID().Str(), sn2.ID().Str()) // get archive stats again @@ -198,13 +196,13 @@ func BenchmarkLoadTree(t *testing.B) { repo := SetupRepo() defer TeardownRepo(repo) - if *benchArchiveDirectory == "" { + if BenchArchiveDirectory == "" { t.Skip("benchdir not set, skipping TestArchiverDedup") } // archive a few files arch := restic.NewArchiver(repo) - sn, _, err := arch.Snapshot(nil, []string{*benchArchiveDirectory}, nil) + sn, _, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil) OK(t, err) t.Logf("archived snapshot %v", sn.ID()) diff --git a/cache_test.go b/cache_test.go index 9e39e86d9..b480b5869 100644 --- a/cache_test.go +++ b/cache_test.go @@ -17,7 +17,7 @@ func TestCache(t *testing.T) { arch := restic.NewArchiver(repo) // archive some files, this should automatically cache all blobs from the snapshot - _, _, err = arch.Snapshot(nil, []string{*benchArchiveDirectory}, nil) + _, _, err = arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil) // TODO: test caching index } diff --git a/chunker/chunker_test.go b/chunker/chunker_test.go index 5c38d2b0d..6a7f30a7d 100644 --- a/chunker/chunker_test.go +++ b/chunker/chunker_test.go @@ -5,12 +5,10 @@ import ( "crypto/md5" "crypto/sha256" "encoding/hex" - "flag" "hash" "io" "io/ioutil" "math/rand" - "os" "testing" "time" @@ -18,8 +16,6 @@ import ( . "github.com/restic/restic/test" ) -var benchmarkFile = flag.String("bench.file", "", "read from this file for benchmark") - func parseDigest(s string) []byte { d, err := hex.DecodeString(s) if err != nil { @@ -247,29 +243,8 @@ func TestChunkerWithoutHash(t *testing.T) { } func benchmarkChunker(b *testing.B, hash hash.Hash) { - var ( - rd io.ReadSeeker - size int - ) - - if *benchmarkFile != "" { - b.Logf("using file %q for benchmark", *benchmarkFile) - f, err := os.Open(*benchmarkFile) - if err != nil { - b.Fatalf("open(%q): %v", *benchmarkFile, err) - } - - fi, err := f.Stat() - if err != nil { - b.Fatalf("lstat(%q): %v", *benchmarkFile, err) - } - - size = int(fi.Size()) - rd = f - } else { - size = 10 * 1024 * 1024 - rd = bytes.NewReader(getRandom(23, size)) - } + size := 10 * 1024 * 1024 + rd := bytes.NewReader(getRandom(23, size)) b.ResetTimer() b.SetBytes(int64(size)) diff --git a/crypto/crypto_test.go b/crypto/crypto_test.go index faba9970c..e8911c086 100644 --- a/crypto/crypto_test.go +++ b/crypto/crypto_test.go @@ -2,7 +2,6 @@ package crypto_test import ( "bytes" - "flag" "io" "io/ioutil" "os" @@ -13,13 +12,13 @@ import ( . "github.com/restic/restic/test" ) -var testLargeCrypto = flag.Bool("test.largecrypto", false, "also test crypto functions with large payloads") +const testLargeCrypto = false func TestEncryptDecrypt(t *testing.T) { k := crypto.NewRandomKey() tests := []int{5, 23, 2<<18 + 23, 1 << 20} - if *testLargeCrypto { + if testLargeCrypto { tests = append(tests, 7<<20+123) } @@ -117,7 +116,7 @@ func TestCornerCases(t *testing.T) { } func TestLargeEncrypt(t *testing.T) { - if !*testLargeCrypto { + if !testLargeCrypto { t.SkipNow() } @@ -252,7 +251,7 @@ func TestEncryptStreamWriter(t *testing.T) { k := crypto.NewRandomKey() tests := []int{5, 23, 2<<18 + 23, 1 << 20} - if *testLargeCrypto { + if testLargeCrypto { tests = append(tests, 7<<20+123) } @@ -286,7 +285,7 @@ func TestDecryptStreamReader(t *testing.T) { k := crypto.NewRandomKey() tests := []int{5, 23, 2<<18 + 23, 1 << 20} - if *testLargeCrypto { + if testLargeCrypto { tests = append(tests, 7<<20+123) } @@ -320,7 +319,7 @@ func TestEncryptWriter(t *testing.T) { k := crypto.NewRandomKey() tests := []int{5, 23, 2<<18 + 23, 1 << 20} - if *testLargeCrypto { + if testLargeCrypto { tests = append(tests, 7<<20+123) } diff --git a/pipe/pipe_test.go b/pipe/pipe_test.go index edb270e6f..42ff7c31d 100644 --- a/pipe/pipe_test.go +++ b/pipe/pipe_test.go @@ -1,7 +1,6 @@ package pipe_test import ( - "flag" "os" "path/filepath" "sync" @@ -12,9 +11,6 @@ import ( . "github.com/restic/restic/test" ) -var testWalkerPath = flag.String("test.walkerpath", ".", "pipeline walker testpath (default: .)") -var maxWorkers = flag.Int("test.workers", 100, "max concurrency (default: 100)") - func isFile(fi os.FileInfo) bool { return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0 } @@ -27,7 +23,7 @@ func statPath(path string) (stats, error) { var s stats // count files and directories with filepath.Walk() - err := filepath.Walk(*testWalkerPath, func(p string, fi os.FileInfo, err error) error { + err := filepath.Walk(TestWalkerPath, func(p string, fi os.FileInfo, err error) error { if fi == nil { return err } @@ -44,15 +40,17 @@ func statPath(path string) (stats, error) { return s, err } +const maxWorkers = 100 + func TestPipelineWalkerWithSplit(t *testing.T) { - if *testWalkerPath == "" { + if TestWalkerPath == "" { t.Skipf("walkerpath not set, skipping TestPipelineWalker") } - before, err := statPath(*testWalkerPath) + before, err := statPath(TestWalkerPath) OK(t, err) - t.Logf("walking path %s with %d dirs, %d files", *testWalkerPath, + t.Logf("walking path %s with %d dirs, %d files", TestWalkerPath, before.dirs, before.files) // account for top level dir @@ -105,7 +103,7 @@ func TestPipelineWalkerWithSplit(t *testing.T) { entCh := make(chan pipe.Entry) dirCh := make(chan pipe.Dir) - for i := 0; i < *maxWorkers; i++ { + for i := 0; i < maxWorkers; i++ { wg.Add(1) go worker(&wg, done, entCh, dirCh) } @@ -120,7 +118,7 @@ func TestPipelineWalkerWithSplit(t *testing.T) { }() resCh := make(chan pipe.Result, 1) - err = pipe.Walk([]string{*testWalkerPath}, done, jobs, resCh) + err = pipe.Walk([]string{TestWalkerPath}, done, jobs, resCh) OK(t, err) // wait for all workers to terminate @@ -129,21 +127,21 @@ func TestPipelineWalkerWithSplit(t *testing.T) { // wait for top-level blob <-resCh - t.Logf("walked path %s with %d dirs, %d files", *testWalkerPath, + t.Logf("walked path %s with %d dirs, %d files", TestWalkerPath, after.dirs, after.files) Assert(t, before == after, "stats do not match, expected %v, got %v", before, after) } func TestPipelineWalker(t *testing.T) { - if *testWalkerPath == "" { + if TestWalkerPath == "" { t.Skipf("walkerpath not set, skipping TestPipelineWalker") } - before, err := statPath(*testWalkerPath) + before, err := statPath(TestWalkerPath) OK(t, err) - t.Logf("walking path %s with %d dirs, %d files", *testWalkerPath, + t.Logf("walking path %s with %d dirs, %d files", TestWalkerPath, before.dirs, before.files) // account for top level dir @@ -194,13 +192,13 @@ func TestPipelineWalker(t *testing.T) { done := make(chan struct{}) jobs := make(chan pipe.Job) - for i := 0; i < *maxWorkers; i++ { + for i := 0; i < maxWorkers; i++ { wg.Add(1) go worker(&wg, done, jobs) } resCh := make(chan pipe.Result, 1) - err = pipe.Walk([]string{*testWalkerPath}, done, jobs, resCh) + err = pipe.Walk([]string{TestWalkerPath}, done, jobs, resCh) OK(t, err) // wait for all workers to terminate @@ -209,14 +207,14 @@ func TestPipelineWalker(t *testing.T) { // wait for top-level blob <-resCh - t.Logf("walked path %s with %d dirs, %d files", *testWalkerPath, + t.Logf("walked path %s with %d dirs, %d files", TestWalkerPath, after.dirs, after.files) Assert(t, before == after, "stats do not match, expected %v, got %v", before, after) } func BenchmarkPipelineWalker(b *testing.B) { - if *testWalkerPath == "" { + if TestWalkerPath == "" { b.Skipf("walkerpath not set, skipping BenchPipelineWalker") } @@ -283,8 +281,8 @@ func BenchmarkPipelineWalker(b *testing.B) { dirCh := make(chan pipe.Dir, 200) var wg sync.WaitGroup - b.Logf("starting %d workers", *maxWorkers) - for i := 0; i < *maxWorkers; i++ { + b.Logf("starting %d workers", maxWorkers) + for i := 0; i < maxWorkers; i++ { wg.Add(2) go dirWorker(&wg, done, dirCh) go fileWorker(&wg, done, entCh) @@ -300,7 +298,7 @@ func BenchmarkPipelineWalker(b *testing.B) { }() resCh := make(chan pipe.Result, 1) - err := pipe.Walk([]string{*testWalkerPath}, done, jobs, resCh) + err := pipe.Walk([]string{TestWalkerPath}, done, jobs, resCh) OK(b, err) // wait for all workers to terminate @@ -314,13 +312,13 @@ func BenchmarkPipelineWalker(b *testing.B) { } func TestPipelineWalkerMultiple(t *testing.T) { - if *testWalkerPath == "" { + if TestWalkerPath == "" { t.Skipf("walkerpath not set, skipping TestPipelineWalker") } - paths, err := filepath.Glob(filepath.Join(*testWalkerPath, "*")) + paths, err := filepath.Glob(filepath.Join(TestWalkerPath, "*")) - before, err := statPath(*testWalkerPath) + before, err := statPath(TestWalkerPath) OK(t, err) t.Logf("walking paths %v with %d dirs, %d files", paths, @@ -371,7 +369,7 @@ func TestPipelineWalkerMultiple(t *testing.T) { done := make(chan struct{}) jobs := make(chan pipe.Job) - for i := 0; i < *maxWorkers; i++ { + for i := 0; i < maxWorkers; i++ { wg.Add(1) go worker(&wg, done, jobs) } diff --git a/repository/repository_test.go b/repository/repository_test.go index 2ccb0b284..7691e596a 100644 --- a/repository/repository_test.go +++ b/repository/repository_test.go @@ -5,7 +5,6 @@ import ( "crypto/rand" "crypto/sha256" "encoding/json" - "flag" "io" "testing" @@ -15,8 +14,6 @@ import ( . "github.com/restic/restic/test" ) -var benchTestDir = flag.String("test.dir", ".", "dir used in benchmarks (default: .)") - type testJSONStruct struct { Foo uint32 Bar string @@ -159,12 +156,12 @@ func TestLoadJSONPack(t *testing.T) { repo := SetupRepo() defer TeardownRepo(repo) - if *benchTestDir == "" { + if BenchArchiveDirectory == "" { t.Skip("benchdir not set, skipping") } // archive a few files - sn := SnapshotDir(t, repo, *benchTestDir, nil) + sn := SnapshotDir(t, repo, BenchArchiveDirectory, nil) OK(t, repo.Flush()) tree := restic.NewTree() @@ -176,7 +173,7 @@ func TestLoadJSONUnpacked(t *testing.T) { repo := SetupRepo() defer TeardownRepo(repo) - if *benchTestDir == "" { + if BenchArchiveDirectory == "" { t.Skip("benchdir not set, skipping") } diff --git a/test/backend.go b/test/backend.go index 60aaa1249..c2d1fe285 100644 --- a/test/backend.go +++ b/test/backend.go @@ -14,12 +14,13 @@ import ( ) var ( - TestPassword = getStringVar("RESTIC_TEST_PASSWORD", "geheim") - TestCleanup = getBoolVar("RESTIC_TEST_CLEANUP", true) - TestTempDir = getStringVar("RESTIC_TEST_TMPDIR", "") - RunIntegrationTest = getBoolVar("RESTIC_TEST_INTEGRATION", true) - TestSFTPPath = getStringVar("RESTIC_TEST_SFTPPATH", - "/usr/lib/ssh:/usr/lib/openssh") + TestPassword = getStringVar("RESTIC_TEST_PASSWORD", "geheim") + TestCleanup = getBoolVar("RESTIC_TEST_CLEANUP", true) + TestTempDir = getStringVar("RESTIC_TEST_TMPDIR", "") + RunIntegrationTest = getBoolVar("RESTIC_TEST_INTEGRATION", true) + TestSFTPPath = getStringVar("RESTIC_TEST_SFTPPATH", "/usr/lib/ssh:/usr/lib/openssh") + TestWalkerPath = getStringVar("RESTIC_TEST_PATH", ".") + BenchArchiveDirectory = getStringVar("RESTIC_BENCH_DIR", ".") ) func getStringVar(name, defaultValue string) string { diff --git a/walk_test.go b/walk_test.go index 15ff87f01..397655978 100644 --- a/walk_test.go +++ b/walk_test.go @@ -1,7 +1,6 @@ package restic_test import ( - "flag" "path/filepath" "testing" @@ -10,13 +9,11 @@ import ( . "github.com/restic/restic/test" ) -var testWalkDirectory = flag.String("test.walkdir", ".", "test walking a directory (globbing pattern, default: .)") - func TestWalkTree(t *testing.T) { repo := SetupRepo() defer TeardownRepo(repo) - dirs, err := filepath.Glob(*testWalkDirectory) + dirs, err := filepath.Glob(TestWalkerPath) OK(t, err) // archive a few files From d51fd436b541567acba42842b0b381400839b496 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Wed, 24 Jun 2015 18:17:01 +0200 Subject: [PATCH 03/12] Add locking functions --- cmd/restic/cmd_cat.go | 14 ++- doc/Design.md | 5 + lock.go | 236 ++++++++++++++++++++++++++++++++++++++++++ lock_test.go | 170 ++++++++++++++++++++++++++++++ 4 files changed, 424 insertions(+), 1 deletion(-) create mode 100644 lock.go create mode 100644 lock_test.go diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index 65345cfe7..4356c1d51 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -126,7 +126,19 @@ func (cmd CmdCat) Execute(args []string) error { fmt.Println(string(buf)) return nil case "lock": - return errors.New("not yet implemented") + lock, err := restic.LoadLock(s, id) + if err != nil { + return err + } + + buf, err := json.MarshalIndent(&lock, "", " ") + if err != nil { + return err + } + + fmt.Println(string(buf)) + + return nil } // load index, handle all the other types diff --git a/doc/Design.md b/doc/Design.md index 483ce3621..8dcab200d 100644 --- a/doc/Design.md +++ b/doc/Design.md @@ -375,6 +375,11 @@ As can be seen from the output of the program `sha256sum`, the hash matches the plaintext hash from the map included in the tree above, so the correct data has been returned. +Locks +----- + + + Backups and Deduplication ========================= diff --git a/lock.go b/lock.go new file mode 100644 index 000000000..07869a28b --- /dev/null +++ b/lock.go @@ -0,0 +1,236 @@ +package restic + +import ( + "errors" + "os" + "os/signal" + "os/user" + "strconv" + "sync" + "syscall" + "time" + + "github.com/restic/restic/backend" + "github.com/restic/restic/debug" + "github.com/restic/restic/repository" +) + +// Lock represents a process locking the repository for an operation. +// +// There are two types of locks: exclusive and non-exclusive. There may be many +// different non-exclusive locks, but at most one exclusive lock, which can +// only be acquired while no non-exclusive lock is held. +type Lock struct { + Time time.Time `json:"time"` + Exclusive bool `json:"exclusive"` + Hostname string `json:"hostname"` + Username string `json:"username"` + PID int `json:"pid"` + UID uint32 `json:"uid,omitempty"` + GID uint32 `json:"gid,omitempty"` + + repo *repository.Repository + lockID backend.ID +} + +var ( + ErrAlreadyLocked = errors.New("already locked") + ErrStaleLockFound = errors.New("stale lock found") +) + +// NewLock returns a new, non-exclusive lock for the repository. If an +// exclusive lock is already held by another process, ErrAlreadyLocked is +// returned. +func NewLock(repo *repository.Repository) (*Lock, error) { + return newLock(repo, false) +} + +// NewExclusiveLock returns a new, exclusive lock for the repository. If +// another lock (normal and exclusive) is already held by another process, +// ErrAlreadyLocked is returned. +func NewExclusiveLock(repo *repository.Repository) (*Lock, error) { + return newLock(repo, true) +} + +func newLock(repo *repository.Repository, excl bool) (*Lock, error) { + lock := &Lock{ + Time: time.Now(), + PID: os.Getpid(), + Exclusive: excl, + repo: repo, + } + + hn, err := os.Hostname() + if err == nil { + lock.Hostname = hn + } + + if err = lock.fillUserInfo(); err != nil { + return nil, err + } + + if err = lock.checkForOtherLocks(); err != nil { + return nil, err + } + + err = lock.createLock() + if err != nil { + return nil, err + } + + return lock, nil +} + +func (l *Lock) fillUserInfo() error { + usr, err := user.Current() + if err != nil { + return nil + } + l.Username = usr.Username + + uid, err := strconv.ParseInt(usr.Uid, 10, 32) + if err != nil { + return err + } + l.UID = uint32(uid) + + gid, err := strconv.ParseInt(usr.Gid, 10, 32) + if err != nil { + return err + } + l.GID = uint32(gid) + + return nil +} + +// checkForOtherLocks looks for other locks that currently exist in the repository. +// +// If an exclusive lock is to be created, checkForOtherLocks returns an error +// if there are any other locks, regardless if exclusive or not. If a +// non-exclusive lock is to be created, an error is only returned when an +// exclusive lock is found. +func (l *Lock) checkForOtherLocks() error { + return eachLock(l.repo, func(id backend.ID, lock *Lock, err error) error { + // ignore locks that cannot be loaded + if err != nil { + return nil + } + + if l.Exclusive { + return ErrAlreadyLocked + } + + if !l.Exclusive && lock.Exclusive { + return ErrAlreadyLocked + } + + return nil + }) +} + +func eachLock(repo *repository.Repository, f func(backend.ID, *Lock, error) error) error { + done := make(chan struct{}) + defer close(done) + + for id := range repo.List(backend.Lock, done) { + lock, err := LoadLock(repo, id) + err = f(id, lock, err) + if err != nil { + return err + } + } + + return nil +} + +// createLock acquires the lock by creating a file in the repository. +func (l *Lock) createLock() error { + id, err := l.repo.SaveJSONUnpacked(backend.Lock, l) + if err != nil { + return err + } + + l.lockID = id + return nil +} + +// Unlock removes the lock from the repository. +func (l *Lock) Unlock() error { + if l == nil || l.lockID == nil { + return nil + } + + return l.repo.Backend().Remove(backend.Lock, l.lockID.String()) +} + +var staleTimeout = 30 * time.Minute + +// Stale returns true if the lock is stale. A lock is stale if the timestamp is +// older than 30 minutes or if it was created on the current machine and the +// process isn't alive any more. +func (l *Lock) Stale() bool { + debug.Log("Lock.Stale", "testing if lock %v for process %d is stale", l.lockID.Str(), l.PID) + if time.Now().Sub(l.Time) > staleTimeout { + debug.Log("Lock.Stale", "lock is stale, timestamp is too old: %v\n", l.Time) + return true + } + + proc, err := os.FindProcess(l.PID) + defer proc.Release() + if err != nil { + debug.Log("Lock.Stale", "error searching for process %d: %v\n", l.PID, err) + return true + } + + debug.Log("Lock.Stale", "sending SIGHUP to process %d\n", l.PID) + err = proc.Signal(syscall.SIGHUP) + if err != nil { + debug.Log("Lock.Stale", "signal error: %v, lock is probably stale\n", err) + return true + } + + debug.Log("Lock.Stale", "lock not stale\n") + return false +} + +// listen for incoming SIGHUP and ignore +var ignoreSIGHUP sync.Once + +func init() { + ignoreSIGHUP.Do(func() { + go func() { + c := make(chan os.Signal) + signal.Notify(c, syscall.SIGHUP) + for s := range c { + debug.Log("lock.ignoreSIGHUP", "Signal received: %v\n", s) + } + }() + }) +} + +// LoadLock loads and unserializes a lock from a repository. +func LoadLock(repo *repository.Repository, id backend.ID) (*Lock, error) { + lock := &Lock{} + if err := repo.LoadJSONUnpacked(backend.Lock, id, lock); err != nil { + return nil, err + } + lock.lockID = id + + return lock, nil +} + +// RemoveStaleLocks deletes all locks detected as stale from the repository. +func RemoveStaleLocks(repo *repository.Repository) error { + return eachLock(repo, func(id backend.ID, lock *Lock, err error) error { + // ignore locks that cannot be loaded + if err != nil { + return nil + } + + if lock.Stale() { + return repo.Backend().Remove(backend.Lock, id.String()) + } + + return nil + }) +} diff --git a/lock_test.go b/lock_test.go new file mode 100644 index 000000000..a3a89c9e1 --- /dev/null +++ b/lock_test.go @@ -0,0 +1,170 @@ +package restic_test + +import ( + "os" + "testing" + "time" + + "github.com/restic/restic" + "github.com/restic/restic/backend" + "github.com/restic/restic/repository" + . "github.com/restic/restic/test" +) + +func TestLock(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + lock, err := restic.NewLock(repo) + OK(t, err) + + OK(t, lock.Unlock()) +} + +func TestDoubleUnlock(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + lock, err := restic.NewLock(repo) + OK(t, err) + + OK(t, lock.Unlock()) + + err = lock.Unlock() + Assert(t, err != nil, + "double unlock didn't return an error, got %v", err) +} + +func TestMultipleLock(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + lock1, err := restic.NewLock(repo) + OK(t, err) + + lock2, err := restic.NewLock(repo) + OK(t, err) + + OK(t, lock1.Unlock()) + OK(t, lock2.Unlock()) +} + +func TestLockExclusive(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + elock, err := restic.NewExclusiveLock(repo) + OK(t, err) + OK(t, elock.Unlock()) +} + +func TestLockOnExclusiveLockedRepo(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + elock, err := restic.NewExclusiveLock(repo) + OK(t, err) + + lock, err := restic.NewLock(repo) + Assert(t, err == restic.ErrAlreadyLocked, + "create normal lock with exclusively locked repo didn't return an error") + + OK(t, lock.Unlock()) + OK(t, elock.Unlock()) +} + +func TestExclusiveLockOnLockedRepo(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + elock, err := restic.NewLock(repo) + OK(t, err) + + lock, err := restic.NewExclusiveLock(repo) + Assert(t, err == restic.ErrAlreadyLocked, + "create exclusive lock with locked repo didn't return an error") + + OK(t, lock.Unlock()) + OK(t, elock.Unlock()) +} + +func createFakeLock(repo *repository.Repository, t time.Time, pid int) (backend.ID, error) { + newLock := &restic.Lock{Time: t, PID: pid} + return repo.SaveJSONUnpacked(backend.Lock, &newLock) +} + +func removeLock(repo *repository.Repository, id backend.ID) error { + return repo.Backend().Remove(backend.Lock, id.String()) +} + +var staleLockTests = []struct { + timestamp time.Time + stale bool + pid int +}{ + { + timestamp: time.Now(), + stale: false, + pid: os.Getpid(), + }, + { + timestamp: time.Now().Add(-time.Hour), + stale: true, + pid: os.Getpid(), + }, + { + timestamp: time.Now().Add(3 * time.Minute), + stale: false, + pid: os.Getpid(), + }, + { + timestamp: time.Now(), + stale: true, + pid: os.Getpid() + 500, + }, +} + +func TestLockStale(t *testing.T) { + for i, test := range staleLockTests { + lock := restic.Lock{ + Time: test.timestamp, + PID: test.pid, + } + + Assert(t, lock.Stale() == test.stale, + "TestStaleLock: test %d failed: expected stale: %v, got %v", + i, test.stale, !test.stale) + } +} + +func lockExists(repo *repository.Repository, t testing.TB, id backend.ID) bool { + exists, err := repo.Backend().Test(backend.Lock, id.String()) + OK(t, err) + + return exists +} + +func TestLockWithStaleLock(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + id1, err := createFakeLock(repo, time.Now().Add(-time.Hour), os.Getpid()) + OK(t, err) + + id2, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()) + OK(t, err) + + id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500) + OK(t, err) + + OK(t, restic.RemoveStaleLocks(repo)) + + Assert(t, lockExists(repo, t, id1) == false, + "stale lock still exists after RemoveStaleLocks was called") + Assert(t, lockExists(repo, t, id2) == true, + "non-stale lock was removed by RemoveStaleLocks") + Assert(t, lockExists(repo, t, id3) == false, + "stale lock still exists after RemoveStaleLocks was called") + + OK(t, removeLock(repo, id2)) +} From a217f51f2ce914eea0138a823cbdc45f33d40abd Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 27 Jun 2015 13:57:52 +0200 Subject: [PATCH 04/12] Add locks to the design document --- doc/Design.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/doc/Design.md b/doc/Design.md index 8dcab200d..e1ce4f275 100644 --- a/doc/Design.md +++ b/doc/Design.md @@ -378,7 +378,43 @@ been returned. Locks ----- +The restic repository structure is designed in a way that allows parallel +access of multiple instance of restic and even parallel writes. However, there +are some functions that work more efficient or even require exclusive access of +the repository. In order to implement these functions, restic processes are +required to create a lock on the repository before doing anything. +Locks come in two types: Exclusive and non-exclusive locks. At most one +process can have an exclusive lock on the repository, and during that time +there mustn't be any other locks (exclusive and non-exclusive). There may be +multiple non-exclusive locks in parallel. + +A lock is a file in the subdir `locks` whose filename is the storage ID of +the contents. It is encrypted and authenticated the same way as other files +in the repository and contains the following JSON structure: + + { + "time": "2015-06-27T12:18:51.759239612+02:00", + "exclusive": false, + "hostname": "kasimir", + "username": "fd0", + "pid": 13607, + "uid": 1000, + "gid": 100 + } + +The field `exclusive` defines the type of lock. When a new lock is to be +created, restic checks all locks in the repository. When a lock is found, it +is tested if the lock is stale, which is the case for locks with timestamps +older than 30 minutes. If the lock was created on the same machine, even for +younger locks it is tested whether the process is still alive by sending a +signal to it. If that fails, restic assumes that the process is dead and +considers the lock to be stale. + +When a new lock is to be created and no other conflicting locks are +detected, restic creates a new lock, waits, and checks if other locks +appeared in the repository. Depending on the type of the other locks and the +lock to be created, restic either continues or fails. Backups and Deduplication ========================= From fba912440dfe1fee85fef923640aa8937cf56de3 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 27 Jun 2015 14:26:33 +0200 Subject: [PATCH 05/12] Add lock conflict check --- lock.go | 13 +++++++++ lock_test.go | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/lock.go b/lock.go index 07869a28b..bf0725b02 100644 --- a/lock.go +++ b/lock.go @@ -52,6 +52,8 @@ func NewExclusiveLock(repo *repository.Repository) (*Lock, error) { return newLock(repo, true) } +const waitBeforeLockCheck = 200 * time.Millisecond + func newLock(repo *repository.Repository, excl bool) (*Lock, error) { lock := &Lock{ Time: time.Now(), @@ -78,6 +80,13 @@ func newLock(repo *repository.Repository, excl bool) (*Lock, error) { return nil, err } + time.Sleep(waitBeforeLockCheck) + + if err = lock.checkForOtherLocks(); err != nil { + lock.Unlock() + return nil, ErrAlreadyLocked + } + return lock, nil } @@ -111,6 +120,10 @@ func (l *Lock) fillUserInfo() error { // exclusive lock is found. func (l *Lock) checkForOtherLocks() error { return eachLock(l.repo, func(id backend.ID, lock *Lock, err error) error { + if id.Equal(l.lockID) { + return nil + } + // ignore locks that cannot be loaded if err != nil { return nil diff --git a/lock_test.go b/lock_test.go index a3a89c9e1..15e48623a 100644 --- a/lock_test.go +++ b/lock_test.go @@ -2,6 +2,7 @@ package restic_test import ( "os" + "sync" "testing" "time" @@ -168,3 +169,79 @@ func TestLockWithStaleLock(t *testing.T) { OK(t, removeLock(repo, id2)) } + +func TestLockConflictingExclusiveLocks(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + for _, jobs := range []int{5, 23, 200} { + var wg sync.WaitGroup + errch := make(chan error, jobs) + + f := func() { + defer wg.Done() + + lock, err := restic.NewExclusiveLock(repo) + errch <- err + OK(t, lock.Unlock()) + } + + for i := 0; i < jobs; i++ { + wg.Add(1) + go f() + } + + errors := 0 + for i := 0; i < jobs; i++ { + err := <-errch + if err != nil { + errors++ + } + } + + wg.Wait() + + Assert(t, errors == jobs-1, + "Expected %d errors, got %d", jobs-1, errors) + } +} + +func TestLockConflictingLocks(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + var wg sync.WaitGroup + + errch := make(chan error, 2) + + wg.Add(2) + + go func() { + defer wg.Done() + + lock, err := restic.NewExclusiveLock(repo) + errch <- err + OK(t, lock.Unlock()) + }() + + go func() { + defer wg.Done() + + lock, err := restic.NewLock(repo) + errch <- err + OK(t, lock.Unlock()) + }() + + errors := 0 + for i := 0; i < 2; i++ { + err := <-errch + if err != nil { + errors++ + } + } + + wg.Wait() + + Assert(t, errors == 1, + "Expected exactly one errors, got %d", errors) +} From 7d2699b42950cc16dad4b13855813842657bf884 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 27 Jun 2015 14:36:46 +0200 Subject: [PATCH 06/12] cmd/restic: Rename variable, no functional changes --- cmd/restic/cmd_backup.go | 10 +++++----- cmd/restic/cmd_cache.go | 6 +++--- cmd/restic/cmd_cat.go | 24 ++++++++++++------------ cmd/restic/cmd_find.go | 10 +++++----- cmd/restic/cmd_fsck.go | 14 +++++++------- cmd/restic/cmd_key.go | 12 ++++++------ cmd/restic/cmd_list.go | 8 ++++---- cmd/restic/cmd_ls.go | 10 +++++----- cmd/restic/cmd_restore.go | 8 ++++---- cmd/restic/cmd_snapshots.go | 8 ++++---- 10 files changed, 55 insertions(+), 55 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 2ab9dc5e0..c377c5cad 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -215,12 +215,12 @@ func (cmd CmdBackup) Execute(args []string) error { target = append(target, d) } - s, err := cmd.global.OpenRepository() + repo, err := cmd.global.OpenRepository() if err != nil { return err } - err = s.LoadIndex() + err = repo.LoadIndex() if err != nil { return err } @@ -229,7 +229,7 @@ func (cmd CmdBackup) Execute(args []string) error { // Force using a parent if !cmd.Force && cmd.Parent != "" { - parentSnapshotID, err = restic.FindSnapshot(s, cmd.Parent) + parentSnapshotID, err = restic.FindSnapshot(repo, cmd.Parent) if err != nil { return fmt.Errorf("invalid id %q: %v", cmd.Parent, err) } @@ -239,7 +239,7 @@ func (cmd CmdBackup) Execute(args []string) error { // Find last snapshot to set it as parent, if not already set if !cmd.Force && parentSnapshotID == nil { - parentSnapshotID, err = findLatestSnapshot(s, target) + parentSnapshotID, err = findLatestSnapshot(repo, target) if err != nil { return err } @@ -258,7 +258,7 @@ func (cmd CmdBackup) Execute(args []string) error { // return true // } - arch := restic.NewArchiver(s) + arch := restic.NewArchiver(repo) arch.Error = func(dir string, fi os.FileInfo, err error) error { // TODO: make ignoring errors configurable diff --git a/cmd/restic/cmd_cache.go b/cmd/restic/cmd_cache.go index 26a6b7970..011fb3e23 100644 --- a/cmd/restic/cmd_cache.go +++ b/cmd/restic/cmd_cache.go @@ -29,18 +29,18 @@ func (cmd CmdCache) Execute(args []string) error { // return fmt.Errorf("wrong number of parameters, Usage: %s", cmd.Usage()) // } - s, err := cmd.global.OpenRepository() + repo, err := cmd.global.OpenRepository() if err != nil { return err } - cache, err := restic.NewCache(s, cmd.global.CacheDir) + cache, err := restic.NewCache(repo, cmd.global.CacheDir) if err != nil { return err } fmt.Printf("clear cache for old snapshots\n") - err = cache.Clear(s) + err = cache.Clear(repo) if err != nil { return err } diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index 4356c1d51..fc4ebc721 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -37,7 +37,7 @@ func (cmd CmdCat) Execute(args []string) error { return fmt.Errorf("type or ID not specified, Usage: %s", cmd.Usage()) } - s, err := cmd.global.OpenRepository() + repo, err := cmd.global.OpenRepository() if err != nil { return err } @@ -55,7 +55,7 @@ func (cmd CmdCat) Execute(args []string) error { } // find snapshot id with prefix - id, err = restic.FindSnapshot(s, args[1]) + id, err = restic.FindSnapshot(repo, args[1]) if err != nil { return err } @@ -65,7 +65,7 @@ func (cmd CmdCat) Execute(args []string) error { // handle all types that don't need an index switch tpe { case "config": - buf, err := json.MarshalIndent(s.Config, "", " ") + buf, err := json.MarshalIndent(repo.Config, "", " ") if err != nil { return err } @@ -73,7 +73,7 @@ func (cmd CmdCat) Execute(args []string) error { fmt.Println(string(buf)) return nil case "index": - buf, err := s.Load(backend.Index, id) + buf, err := repo.Load(backend.Index, id) if err != nil { return err } @@ -83,7 +83,7 @@ func (cmd CmdCat) Execute(args []string) error { case "snapshot": sn := &restic.Snapshot{} - err = s.LoadJSONUnpacked(backend.Snapshot, id, sn) + err = repo.LoadJSONUnpacked(backend.Snapshot, id, sn) if err != nil { return err } @@ -97,7 +97,7 @@ func (cmd CmdCat) Execute(args []string) error { return nil case "key": - rd, err := s.Backend().Get(backend.Key, id.String()) + rd, err := repo.Backend().Get(backend.Key, id.String()) if err != nil { return err } @@ -118,7 +118,7 @@ func (cmd CmdCat) Execute(args []string) error { fmt.Println(string(buf)) return nil case "masterkey": - buf, err := json.MarshalIndent(s.Key(), "", " ") + buf, err := json.MarshalIndent(repo.Key(), "", " ") if err != nil { return err } @@ -126,7 +126,7 @@ func (cmd CmdCat) Execute(args []string) error { fmt.Println(string(buf)) return nil case "lock": - lock, err := restic.LoadLock(s, id) + lock, err := restic.LoadLock(repo, id) if err != nil { return err } @@ -142,14 +142,14 @@ func (cmd CmdCat) Execute(args []string) error { } // load index, handle all the other types - err = s.LoadIndex() + err = repo.LoadIndex() if err != nil { return err } switch tpe { case "pack": - rd, err := s.Backend().Get(backend.Data, id.String()) + rd, err := repo.Backend().Get(backend.Data, id.String()) if err != nil { return err } @@ -158,7 +158,7 @@ func (cmd CmdCat) Execute(args []string) error { return err case "blob": - data, err := s.LoadBlob(pack.Data, id) + data, err := repo.LoadBlob(pack.Data, id) if err == nil { _, err = os.Stdout.Write(data) return err @@ -170,7 +170,7 @@ func (cmd CmdCat) Execute(args []string) error { case "tree": debug.Log("cat", "cat tree %v", id.Str()) tree := restic.NewTree() - err = s.LoadJSONPack(pack.Tree, id, tree) + err = repo.LoadJSONPack(pack.Tree, id, tree) if err != nil { debug.Log("cat", "unable to load tree %v: %v", id.Str(), err) return err diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index c1a56ebc6..103e70cbd 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -157,7 +157,7 @@ func (c CmdFind) Execute(args []string) error { } } - s, err := c.global.OpenRepository() + repo, err := c.global.OpenRepository() if err != nil { return err } @@ -165,18 +165,18 @@ func (c CmdFind) Execute(args []string) error { c.pattern = args[0] if c.Snapshot != "" { - snapshotID, err := restic.FindSnapshot(s, c.Snapshot) + snapshotID, err := restic.FindSnapshot(repo, c.Snapshot) if err != nil { return fmt.Errorf("invalid id %q: %v", args[1], err) } - return c.findInSnapshot(s, snapshotID) + return c.findInSnapshot(repo, snapshotID) } done := make(chan struct{}) defer close(done) - for snapshotID := range s.List(backend.Snapshot, done) { - err := c.findInSnapshot(s, snapshotID) + for snapshotID := range repo.List(backend.Snapshot, done) { + err := c.findInSnapshot(repo, snapshotID) if err != nil { return err diff --git a/cmd/restic/cmd_fsck.go b/cmd/restic/cmd_fsck.go index 17a0608ce..6b8a8e1c6 100644 --- a/cmd/restic/cmd_fsck.go +++ b/cmd/restic/cmd_fsck.go @@ -190,23 +190,23 @@ func (cmd CmdFsck) Execute(args []string) error { cmd.Orphaned = true } - s, err := cmd.global.OpenRepository() + repo, err := cmd.global.OpenRepository() if err != nil { return err } - err = s.LoadIndex() + err = repo.LoadIndex() if err != nil { return err } if cmd.Snapshot != "" { - id, err := restic.FindSnapshot(s, cmd.Snapshot) + id, err := restic.FindSnapshot(repo, cmd.Snapshot) if err != nil { return fmt.Errorf("invalid id %q: %v", cmd.Snapshot, err) } - err = fsckSnapshot(cmd, s, id) + err = fsckSnapshot(cmd, repo, id) if err != nil { fmt.Fprintf(os.Stderr, "check for snapshot %v failed\n", id) } @@ -223,8 +223,8 @@ func (cmd CmdFsck) Execute(args []string) error { defer close(done) var firstErr error - for id := range s.List(backend.Snapshot, done) { - err = fsckSnapshot(cmd, s, id) + for id := range repo.List(backend.Snapshot, done) { + err = fsckSnapshot(cmd, repo, id) if err != nil { fmt.Fprintf(os.Stderr, "check for snapshot %v failed\n", id) firstErr = err @@ -241,7 +241,7 @@ func (cmd CmdFsck) Execute(args []string) error { cnt[pack.Data] = cmd.o_data cnt[pack.Tree] = cmd.o_trees - for blob := range s.Index().Each(done) { + for blob := range repo.Index().Each(done) { debug.Log("restic.fsck", "checking %v blob %v\n", blob.Type, blob.ID) err = cnt[blob.Type].Find(blob.ID) diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index ad1a358fd..19d73c4f6 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -116,25 +116,25 @@ func (cmd CmdKey) Execute(args []string) error { return fmt.Errorf("wrong number of arguments, Usage: %s", cmd.Usage()) } - s, err := cmd.global.OpenRepository() + repo, err := cmd.global.OpenRepository() if err != nil { return err } switch args[0] { case "list": - return cmd.listKeys(s) + return cmd.listKeys(repo) case "add": - return cmd.addKey(s) + return cmd.addKey(repo) case "rm": - id, err := backend.Find(s.Backend(), backend.Key, args[1]) + id, err := backend.Find(repo.Backend(), backend.Key, args[1]) if err != nil { return err } - return cmd.deleteKey(s, id) + return cmd.deleteKey(repo, id) case "passwd": - return cmd.changePassword(s) + return cmd.changePassword(repo) } return nil diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go index bd01e6eda..95fe68db6 100644 --- a/cmd/restic/cmd_list.go +++ b/cmd/restic/cmd_list.go @@ -30,7 +30,7 @@ func (cmd CmdList) Execute(args []string) error { return fmt.Errorf("type not specified, Usage: %s", cmd.Usage()) } - s, err := cmd.global.OpenRepository() + repo, err := cmd.global.OpenRepository() if err != nil { return err } @@ -38,12 +38,12 @@ func (cmd CmdList) Execute(args []string) error { var t backend.Type switch args[0] { case "blobs": - err = s.LoadIndex() + err = repo.LoadIndex() if err != nil { return err } - for blob := range s.Index().Each(nil) { + for blob := range repo.Index().Each(nil) { cmd.global.Printf("%s\n", blob.ID) } @@ -62,7 +62,7 @@ func (cmd CmdList) Execute(args []string) error { return errors.New("invalid type") } - for id := range s.List(t, nil) { + for id := range repo.List(t, nil) { cmd.global.Printf("%s\n", id) } diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 59144befc..8ec904bbd 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -69,27 +69,27 @@ func (cmd CmdLs) Execute(args []string) error { return fmt.Errorf("wrong number of arguments, Usage: %s", cmd.Usage()) } - s, err := cmd.global.OpenRepository() + repo, err := cmd.global.OpenRepository() if err != nil { return err } - err = s.LoadIndex() + err = repo.LoadIndex() if err != nil { return err } - id, err := restic.FindSnapshot(s, args[0]) + id, err := restic.FindSnapshot(repo, args[0]) if err != nil { return err } - sn, err := restic.LoadSnapshot(s, id) + sn, err := restic.LoadSnapshot(repo, id) if err != nil { return err } fmt.Printf("snapshot of %v at %s:\n", sn.Paths, sn.Time) - return printTree("", s, sn.Tree) + return printTree("", repo, sn.Tree) } diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 1f3697d4d..f70d2fc6a 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -30,17 +30,17 @@ func (cmd CmdRestore) Execute(args []string) error { return fmt.Errorf("wrong number of arguments, Usage: %s", cmd.Usage()) } - s, err := cmd.global.OpenRepository() + repo, err := cmd.global.OpenRepository() if err != nil { return err } - err = s.LoadIndex() + err = repo.LoadIndex() if err != nil { return err } - id, err := restic.FindSnapshot(s, args[0]) + id, err := restic.FindSnapshot(repo, args[0]) if err != nil { cmd.global.Exitf(1, "invalid id %q: %v", args[0], err) } @@ -48,7 +48,7 @@ func (cmd CmdRestore) Execute(args []string) error { target := args[1] // create restorer - res, err := restic.NewRestorer(s, id) + res, err := restic.NewRestorer(repo, id) if err != nil { cmd.global.Exitf(2, "creating restorer failed: %v\n", err) } diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index f1f8ba31e..2ffb1d8b1 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -94,7 +94,7 @@ func (cmd CmdSnapshots) Execute(args []string) error { return fmt.Errorf("wrong number of arguments, usage: %s", cmd.Usage()) } - s, err := cmd.global.OpenRepository() + repo, err := cmd.global.OpenRepository() if err != nil { return err } @@ -107,8 +107,8 @@ func (cmd CmdSnapshots) Execute(args []string) error { defer close(done) list := []*restic.Snapshot{} - for id := range s.List(backend.Snapshot, done) { - sn, err := restic.LoadSnapshot(s, id) + for id := range repo.List(backend.Snapshot, done) { + sn, err := restic.LoadSnapshot(repo, id) if err != nil { fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err) continue @@ -127,7 +127,7 @@ func (cmd CmdSnapshots) Execute(args []string) error { } } - plen, err := s.PrefixLength(backend.Snapshot) + plen, err := repo.PrefixLength(backend.Snapshot) if err != nil { return err } From 65a0def9491f8f3a3d84dc588d86f860dc78114d Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 27 Jun 2015 14:40:18 +0200 Subject: [PATCH 07/12] cmd/restic: Add locks to commands --- cmd/restic/cmd_backup.go | 6 ++++++ cmd/restic/cmd_cache.go | 6 ++++++ cmd/restic/cmd_cat.go | 6 ++++++ cmd/restic/cmd_dump.go | 6 ++++++ cmd/restic/cmd_find.go | 6 ++++++ cmd/restic/cmd_fsck.go | 6 ++++++ cmd/restic/cmd_key.go | 7 +++++++ cmd/restic/cmd_list.go | 7 +++++++ cmd/restic/cmd_restore.go | 6 ++++++ cmd/restic/cmd_snapshots.go | 6 ++++++ 10 files changed, 62 insertions(+) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index c377c5cad..a20c4ebc7 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -220,6 +220,12 @@ func (cmd CmdBackup) Execute(args []string) error { return err } + lock, err := restic.NewLock(repo) + defer lock.Unlock() + if err != nil { + return err + } + err = repo.LoadIndex() if err != nil { return err diff --git a/cmd/restic/cmd_cache.go b/cmd/restic/cmd_cache.go index 011fb3e23..de0efddb1 100644 --- a/cmd/restic/cmd_cache.go +++ b/cmd/restic/cmd_cache.go @@ -34,6 +34,12 @@ func (cmd CmdCache) Execute(args []string) error { return err } + lock, err := restic.NewLock(repo) + defer lock.Unlock() + if err != nil { + return err + } + cache, err := restic.NewCache(repo, cmd.global.CacheDir) if err != nil { return err diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index fc4ebc721..36b4aa49a 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -42,6 +42,12 @@ func (cmd CmdCat) Execute(args []string) error { return err } + lock, err := restic.NewLock(repo) + defer lock.Unlock() + if err != nil { + return err + } + tpe := args[0] var id backend.ID diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index b80e61509..e0c4ea702 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -109,6 +109,12 @@ func (cmd CmdDump) Execute(args []string) error { return err } + lock, err := restic.NewLock(repo) + defer lock.Unlock() + if err != nil { + return err + } + err = repo.LoadIndex() if err != nil { return err diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 103e70cbd..4e28e2041 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -162,6 +162,12 @@ func (c CmdFind) Execute(args []string) error { return err } + lock, err := restic.NewLock(repo) + defer lock.Unlock() + if err != nil { + return err + } + c.pattern = args[0] if c.Snapshot != "" { diff --git a/cmd/restic/cmd_fsck.go b/cmd/restic/cmd_fsck.go index 6b8a8e1c6..76e0a1f8b 100644 --- a/cmd/restic/cmd_fsck.go +++ b/cmd/restic/cmd_fsck.go @@ -195,6 +195,12 @@ func (cmd CmdFsck) Execute(args []string) error { return err } + lock, err := restic.NewExclusiveLock(repo) + defer lock.Unlock() + if err != nil { + return err + } + err = repo.LoadIndex() if err != nil { return err diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index 19d73c4f6..47f5333c4 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "github.com/restic/restic" "github.com/restic/restic/backend" "github.com/restic/restic/repository" ) @@ -121,6 +122,12 @@ func (cmd CmdKey) Execute(args []string) error { return err } + lock, err := restic.NewExclusiveLock(repo) + defer lock.Unlock() + if err != nil { + return err + } + switch args[0] { case "list": return cmd.listKeys(repo) diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go index 95fe68db6..dda14aca7 100644 --- a/cmd/restic/cmd_list.go +++ b/cmd/restic/cmd_list.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" + "github.com/restic/restic" "github.com/restic/restic/backend" ) @@ -35,6 +36,12 @@ func (cmd CmdList) Execute(args []string) error { return err } + lock, err := restic.NewLock(repo) + defer lock.Unlock() + if err != nil { + return err + } + var t backend.Type switch args[0] { case "blobs": diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index f70d2fc6a..e2e7344a8 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -35,6 +35,12 @@ func (cmd CmdRestore) Execute(args []string) error { return err } + lock, err := restic.NewLock(repo) + defer lock.Unlock() + if err != nil { + return err + } + err = repo.LoadIndex() if err != nil { return err diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index 2ffb1d8b1..1947ed476 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -99,6 +99,12 @@ func (cmd CmdSnapshots) Execute(args []string) error { return err } + lock, err := restic.NewLock(repo) + defer lock.Unlock() + if err != nil { + return err + } + tab := NewTable() tab.Header = fmt.Sprintf("%-8s %-19s %-10s %s", "ID", "Date", "Source", "Directory") tab.RowFormat = "%-8s %-19s %-10s %s" From 13e9a35f9684b981e5f14de9f378342497db6f72 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 27 Jun 2015 15:05:20 +0200 Subject: [PATCH 08/12] cmd/restic: Add lock handling, interrupt cleanup --- cmd/restic/cmd_backup.go | 4 +- cmd/restic/cmd_cache.go | 4 +- cmd/restic/cmd_cat.go | 4 +- cmd/restic/cmd_dump.go | 4 +- cmd/restic/cmd_find.go | 4 +- cmd/restic/cmd_fsck.go | 4 +- cmd/restic/cmd_key.go | 5 +-- cmd/restic/cmd_list.go | 5 +-- cmd/restic/cmd_restore.go | 4 +- cmd/restic/cmd_snapshots.go | 4 +- cmd/restic/lock.go | 80 +++++++++++++++++++++++++++++++++++++ 11 files changed, 100 insertions(+), 22 deletions(-) create mode 100644 cmd/restic/lock.go diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index a20c4ebc7..32d486329 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -220,8 +220,8 @@ func (cmd CmdBackup) Execute(args []string) error { return err } - lock, err := restic.NewLock(repo) - defer lock.Unlock() + lock, err := lockRepo(repo) + defer unlockRepo(lock) if err != nil { return err } diff --git a/cmd/restic/cmd_cache.go b/cmd/restic/cmd_cache.go index de0efddb1..39733b997 100644 --- a/cmd/restic/cmd_cache.go +++ b/cmd/restic/cmd_cache.go @@ -34,8 +34,8 @@ func (cmd CmdCache) Execute(args []string) error { return err } - lock, err := restic.NewLock(repo) - defer lock.Unlock() + lock, err := lockRepo(repo) + defer unlockRepo(lock) if err != nil { return err } diff --git a/cmd/restic/cmd_cat.go b/cmd/restic/cmd_cat.go index 36b4aa49a..6f22670ea 100644 --- a/cmd/restic/cmd_cat.go +++ b/cmd/restic/cmd_cat.go @@ -42,8 +42,8 @@ func (cmd CmdCat) Execute(args []string) error { return err } - lock, err := restic.NewLock(repo) - defer lock.Unlock() + lock, err := lockRepo(repo) + defer unlockRepo(lock) if err != nil { return err } diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index e0c4ea702..9a12ea308 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -109,8 +109,8 @@ func (cmd CmdDump) Execute(args []string) error { return err } - lock, err := restic.NewLock(repo) - defer lock.Unlock() + lock, err := lockRepo(repo) + defer unlockRepo(lock) if err != nil { return err } diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 4e28e2041..33b91fe43 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -162,8 +162,8 @@ func (c CmdFind) Execute(args []string) error { return err } - lock, err := restic.NewLock(repo) - defer lock.Unlock() + lock, err := lockRepo(repo) + defer unlockRepo(lock) if err != nil { return err } diff --git a/cmd/restic/cmd_fsck.go b/cmd/restic/cmd_fsck.go index 76e0a1f8b..b4e8d3e0d 100644 --- a/cmd/restic/cmd_fsck.go +++ b/cmd/restic/cmd_fsck.go @@ -195,8 +195,8 @@ func (cmd CmdFsck) Execute(args []string) error { return err } - lock, err := restic.NewExclusiveLock(repo) - defer lock.Unlock() + lock, err := lockRepoExclusive(repo) + defer unlockRepo(lock) if err != nil { return err } diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index 47f5333c4..a4a3d7366 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" - "github.com/restic/restic" "github.com/restic/restic/backend" "github.com/restic/restic/repository" ) @@ -122,8 +121,8 @@ func (cmd CmdKey) Execute(args []string) error { return err } - lock, err := restic.NewExclusiveLock(repo) - defer lock.Unlock() + lock, err := lockRepoExclusive(repo) + defer unlockRepo(lock) if err != nil { return err } diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go index dda14aca7..c8ea6c65c 100644 --- a/cmd/restic/cmd_list.go +++ b/cmd/restic/cmd_list.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" - "github.com/restic/restic" "github.com/restic/restic/backend" ) @@ -36,8 +35,8 @@ func (cmd CmdList) Execute(args []string) error { return err } - lock, err := restic.NewLock(repo) - defer lock.Unlock() + lock, err := lockRepo(repo) + defer unlockRepo(lock) if err != nil { return err } diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index e2e7344a8..393d0347d 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -35,8 +35,8 @@ func (cmd CmdRestore) Execute(args []string) error { return err } - lock, err := restic.NewLock(repo) - defer lock.Unlock() + lock, err := lockRepo(repo) + defer unlockRepo(lock) if err != nil { return err } diff --git a/cmd/restic/cmd_snapshots.go b/cmd/restic/cmd_snapshots.go index 1947ed476..38e2046ad 100644 --- a/cmd/restic/cmd_snapshots.go +++ b/cmd/restic/cmd_snapshots.go @@ -99,8 +99,8 @@ func (cmd CmdSnapshots) Execute(args []string) error { return err } - lock, err := restic.NewLock(repo) - defer lock.Unlock() + lock, err := lockRepo(repo) + defer unlockRepo(lock) if err != nil { return err } diff --git a/cmd/restic/lock.go b/cmd/restic/lock.go new file mode 100644 index 000000000..e7d6b9976 --- /dev/null +++ b/cmd/restic/lock.go @@ -0,0 +1,80 @@ +package main + +import ( + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/restic/restic" + "github.com/restic/restic/debug" + "github.com/restic/restic/repository" +) + +var globalLocks []*restic.Lock + +func lockRepo(repo *repository.Repository) (*restic.Lock, error) { + lock, err := restic.NewLock(repo) + if err != nil { + return nil, err + } + + globalLocks = append(globalLocks, lock) + + return lock, err +} + +func lockRepoExclusive(repo *repository.Repository) (*restic.Lock, error) { + lock, err := restic.NewExclusiveLock(repo) + if err != nil { + return nil, err + } + + globalLocks = append(globalLocks, lock) + + return lock, err +} + +func unlockRepo(lock *restic.Lock) error { + if err := lock.Unlock(); err != nil { + return err + } + + for i := 0; i < len(globalLocks); i++ { + if lock == globalLocks[i] { + globalLocks = append(globalLocks[:i], globalLocks[i+1:]...) + return nil + } + } + + return nil +} + +func unlockAll() error { + debug.Log("unlockAll", "unlocking %d locks", len(globalLocks)) + for _, lock := range globalLocks { + if err := lock.Unlock(); err != nil { + return err + } + } + + return nil +} + +func init() { + c := make(chan os.Signal) + signal.Notify(c, syscall.SIGINT) + + go CleanupHandler(c) +} + +// CleanupHandler handles the SIGINT signal. +func CleanupHandler(c <-chan os.Signal) { + for s := range c { + debug.Log("CleanupHandler", "signal %v received, cleaning up", s) + fmt.Println("\x1b[2KInterrupt received, cleaning up") + unlockAll() + fmt.Println("exiting") + os.Exit(0) + } +} From 47212dde8ca634c2a2c805bffdda82a4d2379d1b Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 27 Jun 2015 15:06:41 +0200 Subject: [PATCH 09/12] cmd/restic: Do not require exclusive lock for listing keys --- cmd/restic/cmd_key.go | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index a4a3d7366..c585ae4cd 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -121,18 +121,30 @@ func (cmd CmdKey) Execute(args []string) error { return err } - lock, err := lockRepoExclusive(repo) - defer unlockRepo(lock) - if err != nil { - return err - } - switch args[0] { case "list": + lock, err := lockRepo(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + return cmd.listKeys(repo) case "add": + lock, err := lockRepo(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + return cmd.addKey(repo) case "rm": + lock, err := lockRepoExclusive(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + id, err := backend.Find(repo.Backend(), backend.Key, args[1]) if err != nil { return err @@ -140,6 +152,12 @@ func (cmd CmdKey) Execute(args []string) error { return cmd.deleteKey(repo, id) case "passwd": + lock, err := lockRepoExclusive(repo) + defer unlockRepo(lock) + if err != nil { + return err + } + return cmd.changePassword(repo) } From 0ad3d71f01007b131f2a920a5e34ef946bf92a0d Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 27 Jun 2015 15:47:29 +0200 Subject: [PATCH 10/12] repository: Add more debug to Create() --- repository/repository.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/repository/repository.go b/repository/repository.go index aad5f3c9b..491ef6bd7 100644 --- a/repository/repository.go +++ b/repository/repository.go @@ -372,7 +372,7 @@ func (s *Repository) SaveJSONUnpacked(t backend.Type, item interface{}) (backend if err != nil { return nil, err } - debug.Log("Repo.SaveJSONUnpacked", "create new file %p", blob) + debug.Log("Repo.SaveJSONUnpacked", "create new blob %v", t) // hash hw := backend.NewHashingWriter(blob, sha256.New()) @@ -396,9 +396,12 @@ func (s *Repository) SaveJSONUnpacked(t backend.Type, item interface{}) (backend err = blob.Finalize(t, sid.String()) if err != nil { + debug.Log("Repo.SaveJSONUnpacked", "error saving blob %v as %v: %v", t, sid, err) return nil, err } + debug.Log("Repo.SaveJSONUnpacked", "new blob %v saved as %v", t, sid) + return sid, nil } From e657287eac71b907144d2a0884d454f36157e963 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 27 Jun 2015 15:50:36 +0200 Subject: [PATCH 11/12] cmd/restic: Add command `unlock`, improve error message --- cmd/restic/cmd_unlock.go | 43 +++++++++++++++++++++++++++++++++++ cmd/restic/lock.go | 31 +++++++++++++++++-------- lock.go | 49 +++++++++++++++++++++++++++++++++------- lock_test.go | 33 ++++++++++++++++++++++++--- 4 files changed, 136 insertions(+), 20 deletions(-) create mode 100644 cmd/restic/cmd_unlock.go diff --git a/cmd/restic/cmd_unlock.go b/cmd/restic/cmd_unlock.go new file mode 100644 index 000000000..47345350c --- /dev/null +++ b/cmd/restic/cmd_unlock.go @@ -0,0 +1,43 @@ +package main + +import "github.com/restic/restic" + +type CmdUnlock struct { + RemoveAll bool `long:"remove-all" description:"Remove all locks, even stale ones"` + + global *GlobalOptions +} + +func init() { + _, err := parser.AddCommand("unlock", + "remove locks", + "The unlock command checks for stale locks and removes them", + &CmdUnlock{global: &globalOpts}) + if err != nil { + panic(err) + } +} + +func (cmd CmdUnlock) Usage() string { + return "[unlock-options]" +} + +func (cmd CmdUnlock) Execute(args []string) error { + repo, err := cmd.global.OpenRepository() + if err != nil { + return err + } + + fn := restic.RemoveStaleLocks + if cmd.RemoveAll { + fn = restic.RemoveAllLocks + } + + err = fn(repo) + if err != nil { + return err + } + + cmd.global.Verbosef("successfully removed locks\n") + return nil +} diff --git a/cmd/restic/lock.go b/cmd/restic/lock.go index e7d6b9976..5f6c18802 100644 --- a/cmd/restic/lock.go +++ b/cmd/restic/lock.go @@ -14,19 +14,32 @@ import ( var globalLocks []*restic.Lock func lockRepo(repo *repository.Repository) (*restic.Lock, error) { - lock, err := restic.NewLock(repo) - if err != nil { - return nil, err - } - - globalLocks = append(globalLocks, lock) - - return lock, err + return lockRepository(repo, false) } func lockRepoExclusive(repo *repository.Repository) (*restic.Lock, error) { - lock, err := restic.NewExclusiveLock(repo) + return lockRepository(repo, true) +} + +func lockRepository(repo *repository.Repository, exclusive bool) (*restic.Lock, error) { + lockFn := restic.NewLock + if exclusive { + lockFn = restic.NewExclusiveLock + } + + lock, err := lockFn(repo) if err != nil { + if restic.IsAlreadyLocked(err) { + tpe := "" + if exclusive { + tpe = " exclusive" + } + fmt.Fprintf(os.Stderr, "unable to acquire%s lock for operation:\n", tpe) + fmt.Fprintln(os.Stderr, err) + fmt.Fprintf(os.Stderr, "\nthe `unlock` command can be used to remove stale locks\n") + os.Exit(1) + } + return nil, err } diff --git a/lock.go b/lock.go index bf0725b02..65d102323 100644 --- a/lock.go +++ b/lock.go @@ -1,7 +1,7 @@ package restic import ( - "errors" + "fmt" "os" "os/signal" "os/user" @@ -33,10 +33,24 @@ type Lock struct { lockID backend.ID } -var ( - ErrAlreadyLocked = errors.New("already locked") - ErrStaleLockFound = errors.New("stale lock found") -) +// ErrAlreadyLocked is returned when NewLock or NewExclusiveLock are unable to +// acquire the desired lock. +type ErrAlreadyLocked struct { + otherLock *Lock +} + +func (e ErrAlreadyLocked) Error() string { + return fmt.Sprintf("repository is already locked by %v", e.otherLock) +} + +// IsAlreadyLocked returns true iff err is an instance of ErrAlreadyLocked. +func IsAlreadyLocked(err error) bool { + if _, ok := err.(ErrAlreadyLocked); ok { + return true + } + + return false +} // NewLock returns a new, non-exclusive lock for the repository. If an // exclusive lock is already held by another process, ErrAlreadyLocked is @@ -84,7 +98,7 @@ func newLock(repo *repository.Repository, excl bool) (*Lock, error) { if err = lock.checkForOtherLocks(); err != nil { lock.Unlock() - return nil, ErrAlreadyLocked + return nil, err } return lock, nil @@ -130,11 +144,11 @@ func (l *Lock) checkForOtherLocks() error { } if l.Exclusive { - return ErrAlreadyLocked + return ErrAlreadyLocked{otherLock: lock} } if !l.Exclusive && lock.Exclusive { - return ErrAlreadyLocked + return ErrAlreadyLocked{otherLock: lock} } return nil @@ -206,6 +220,19 @@ func (l *Lock) Stale() bool { return false } +func (l Lock) String() string { + text := fmt.Sprintf("PID %d on %s by %s (UID %d, GID %d)\nlock was created at %s (%s ago)\nstorage ID %v", + l.PID, l.Hostname, l.Username, l.UID, l.GID, + l.Time.Format("2006-01-02 15:04:05"), time.Since(l.Time), + l.lockID.Str()) + + if l.Stale() { + text += " (stale)" + } + + return text +} + // listen for incoming SIGHUP and ignore var ignoreSIGHUP sync.Once @@ -247,3 +274,9 @@ func RemoveStaleLocks(repo *repository.Repository) error { return nil }) } + +func RemoveAllLocks(repo *repository.Repository) error { + return eachLock(repo, func(id backend.ID, lock *Lock, err error) error { + return repo.Backend().Remove(backend.Lock, id.String()) + }) +} diff --git a/lock_test.go b/lock_test.go index 15e48623a..fa4c6bbc4 100644 --- a/lock_test.go +++ b/lock_test.go @@ -67,8 +67,10 @@ func TestLockOnExclusiveLockedRepo(t *testing.T) { OK(t, err) lock, err := restic.NewLock(repo) - Assert(t, err == restic.ErrAlreadyLocked, + Assert(t, err != nil, "create normal lock with exclusively locked repo didn't return an error") + Assert(t, restic.IsAlreadyLocked(err), + "create normal lock with exclusively locked repo didn't return the correct error") OK(t, lock.Unlock()) OK(t, elock.Unlock()) @@ -82,8 +84,10 @@ func TestExclusiveLockOnLockedRepo(t *testing.T) { OK(t, err) lock, err := restic.NewExclusiveLock(repo) - Assert(t, err == restic.ErrAlreadyLocked, - "create exclusive lock with locked repo didn't return an error") + Assert(t, err != nil, + "create normal lock with exclusively locked repo didn't return an error") + Assert(t, restic.IsAlreadyLocked(err), + "create normal lock with exclusively locked repo didn't return the correct error") OK(t, lock.Unlock()) OK(t, elock.Unlock()) @@ -170,6 +174,29 @@ func TestLockWithStaleLock(t *testing.T) { OK(t, removeLock(repo, id2)) } +func TestRemoveAllLocks(t *testing.T) { + repo := SetupRepo() + defer TeardownRepo(repo) + + id1, err := createFakeLock(repo, time.Now().Add(-time.Hour), os.Getpid()) + OK(t, err) + + id2, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()) + OK(t, err) + + id3, err := createFakeLock(repo, time.Now().Add(-time.Minute), os.Getpid()+500) + OK(t, err) + + OK(t, restic.RemoveAllLocks(repo)) + + Assert(t, lockExists(repo, t, id1) == false, + "lock still exists after RemoveAllLocks was called") + Assert(t, lockExists(repo, t, id2) == false, + "lock still exists after RemoveAllLocks was called") + Assert(t, lockExists(repo, t, id3) == false, + "lock still exists after RemoveAllLocks was called") +} + func TestLockConflictingExclusiveLocks(t *testing.T) { repo := SetupRepo() defer TeardownRepo(repo) From 0f09a7e46e3e4873a663b8f8254e4aff48c0f41d Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 28 Jun 2015 16:27:17 +0200 Subject: [PATCH 12/12] Remove unreliable tests --- lock_test.go | 77 ---------------------------------------------------- 1 file changed, 77 deletions(-) diff --git a/lock_test.go b/lock_test.go index fa4c6bbc4..988c5d440 100644 --- a/lock_test.go +++ b/lock_test.go @@ -2,7 +2,6 @@ package restic_test import ( "os" - "sync" "testing" "time" @@ -196,79 +195,3 @@ func TestRemoveAllLocks(t *testing.T) { Assert(t, lockExists(repo, t, id3) == false, "lock still exists after RemoveAllLocks was called") } - -func TestLockConflictingExclusiveLocks(t *testing.T) { - repo := SetupRepo() - defer TeardownRepo(repo) - - for _, jobs := range []int{5, 23, 200} { - var wg sync.WaitGroup - errch := make(chan error, jobs) - - f := func() { - defer wg.Done() - - lock, err := restic.NewExclusiveLock(repo) - errch <- err - OK(t, lock.Unlock()) - } - - for i := 0; i < jobs; i++ { - wg.Add(1) - go f() - } - - errors := 0 - for i := 0; i < jobs; i++ { - err := <-errch - if err != nil { - errors++ - } - } - - wg.Wait() - - Assert(t, errors == jobs-1, - "Expected %d errors, got %d", jobs-1, errors) - } -} - -func TestLockConflictingLocks(t *testing.T) { - repo := SetupRepo() - defer TeardownRepo(repo) - - var wg sync.WaitGroup - - errch := make(chan error, 2) - - wg.Add(2) - - go func() { - defer wg.Done() - - lock, err := restic.NewExclusiveLock(repo) - errch <- err - OK(t, lock.Unlock()) - }() - - go func() { - defer wg.Done() - - lock, err := restic.NewLock(repo) - errch <- err - OK(t, lock.Unlock()) - }() - - errors := 0 - for i := 0; i < 2; i++ { - err := <-errch - if err != nil { - errors++ - } - } - - wg.Wait() - - Assert(t, errors == 1, - "Expected exactly one errors, got %d", errors) -}