From 30ab03b7b73c7e134c2ff4b5842814bb047d7634 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Tue, 23 Sep 2014 22:39:12 +0200 Subject: [PATCH] Add decrypt, refactor --- archiver.go | 189 ++++++++++++++++++ backend/doc.go | 2 + backend/generic.go | 84 ++++++++ backend/generic_test.go | 36 ++++ id.go => backend/id.go | 41 +++- backend/interface.go | 21 ++ backend/local.go | 213 ++++++++++++++++++++ backend/local_test.go | 171 ++++++++++++++++ cmd/archive/main.go | 81 ++++++++ cmd/cat/main.go | 126 ++++++++++++ cmd/decrypt/main.go | 233 ++++++++++++++++++++++ cmd/dirdiff/main.go | 3 + cmd/khepri/cmd_backup.go | 33 ++- cmd/khepri/cmd_dump.go | 90 --------- cmd/khepri/cmd_fsck.go | 116 +++++------ cmd/khepri/cmd_init.go | 20 +- cmd/khepri/cmd_list.go | 28 +-- cmd/khepri/cmd_restore.go | 38 +++- cmd/khepri/cmd_snapshots.go | 37 ++-- cmd/khepri/main.go | 48 ++++- cmd/list/main.go | 110 ++++++++++ contenthandler.go | 242 ++++++++++++++++++++++ key.go | 388 ++++++++++++++++++++++++++++++++++++ key_int_test.go | 79 ++++++++ key_test.go | 50 +++++ object.go | 27 --- object_test.go | 27 --- repository.go | 326 ------------------------------ repository_test.go | 130 ------------ restorer.go | 100 ++++++++++ snapshot.go | 87 ++------ snapshot_test.go | 25 +-- storagemap.go | 51 +++++ test/run.sh | 1 + test/test-backup.sh | 4 +- tree.go | 307 ++++++++-------------------- tree_test.go | 54 +++++ 37 files changed, 2572 insertions(+), 1046 deletions(-) create mode 100644 archiver.go create mode 100644 backend/doc.go create mode 100644 backend/generic.go create mode 100644 backend/generic_test.go rename id.go => backend/id.go (66%) create mode 100644 backend/interface.go create mode 100644 backend/local.go create mode 100644 backend/local_test.go create mode 100644 cmd/archive/main.go create mode 100644 cmd/cat/main.go create mode 100644 cmd/decrypt/main.go delete mode 100644 cmd/khepri/cmd_dump.go create mode 100644 cmd/list/main.go create mode 100644 contenthandler.go create mode 100644 key.go create mode 100644 key_int_test.go create mode 100644 key_test.go delete mode 100644 object.go delete mode 100644 object_test.go delete mode 100644 repository.go delete mode 100644 repository_test.go create mode 100644 restorer.go create mode 100644 storagemap.go create mode 100644 tree_test.go diff --git a/archiver.go b/archiver.go new file mode 100644 index 000000000..15dbb3f43 --- /dev/null +++ b/archiver.go @@ -0,0 +1,189 @@ +package khepri + +import ( + "os" + "path/filepath" + + "github.com/fd0/khepri/backend" +) + +type Archiver struct { + be backend.Server + key *Key + ch *ContentHandler + smap *StorageMap // blobs used for the current snapshot + + Error func(dir string, fi os.FileInfo, err error) error + Filter func(item string, fi os.FileInfo) bool +} + +func NewArchiver(be backend.Server, key *Key) (*Archiver, error) { + var err error + arch := &Archiver{be: be, key: key} + + // abort on all errors + arch.Error = func(string, os.FileInfo, error) error { return err } + // allow all files + arch.Filter = func(string, os.FileInfo) bool { return true } + + arch.smap = NewStorageMap() + arch.ch, err = NewContentHandler(be, key) + if err != nil { + return nil, err + } + + // load all blobs from all snapshots + err = arch.ch.LoadAllSnapshots() + if err != nil { + return nil, err + } + + return arch, nil +} + +func (arch *Archiver) Save(t backend.Type, data []byte) (*Blob, error) { + blob, err := arch.ch.Save(t, data) + if err != nil { + return nil, err + } + + // store blob in storage map for current snapshot + arch.smap.Insert(blob) + + return blob, nil +} + +func (arch *Archiver) SaveJSON(t backend.Type, item interface{}) (*Blob, error) { + blob, err := arch.ch.SaveJSON(t, item) + if err != nil { + return nil, err + } + + // store blob in storage map for current snapshot + arch.smap.Insert(blob) + + return blob, nil +} + +func (arch *Archiver) SaveFile(node *Node) (Blobs, error) { + blobs, err := arch.ch.SaveFile(node.path, uint(node.Size)) + if err != nil { + return nil, arch.Error(node.path, nil, err) + } + + node.Content = make([]backend.ID, len(blobs)) + for i, blob := range blobs { + node.Content[i] = blob.ID + arch.smap.Insert(blob) + } + + return blobs, err +} + +func (arch *Archiver) ImportDir(dir string) (Tree, error) { + fd, err := os.Open(dir) + defer fd.Close() + if err != nil { + return nil, arch.Error(dir, nil, err) + } + + entries, err := fd.Readdir(-1) + if err != nil { + return nil, arch.Error(dir, nil, err) + } + + if len(entries) == 0 { + return nil, nil + } + + tree := Tree{} + + for _, entry := range entries { + path := filepath.Join(dir, entry.Name()) + + if !arch.Filter(path, entry) { + continue + } + + node, err := NodeFromFileInfo(path, entry) + if err != nil { + return nil, arch.Error(dir, entry, err) + } + + tree = append(tree, node) + + if entry.IsDir() { + subtree, err := arch.ImportDir(path) + if err != nil { + return nil, err + } + + blob, err := arch.SaveJSON(backend.Tree, subtree) + if err != nil { + return nil, err + } + + node.Subtree = blob.ID + + continue + } + + if node.Type == "file" { + _, err := arch.SaveFile(node) + if err != nil { + return nil, arch.Error(path, entry, err) + } + } + } + + return tree, nil +} + +func (arch *Archiver) Import(dir string) (*Snapshot, *Blob, error) { + sn := NewSnapshot(dir) + + fi, err := os.Lstat(dir) + if err != nil { + return nil, nil, err + } + + node, err := NodeFromFileInfo(dir, fi) + if err != nil { + return nil, nil, err + } + + if node.Type == "dir" { + tree, err := arch.ImportDir(dir) + if err != nil { + return nil, nil, err + } + + blob, err := arch.SaveJSON(backend.Tree, tree) + if err != nil { + return nil, nil, err + } + + node.Subtree = blob.ID + } else if node.Type == "file" { + _, err := arch.SaveFile(node) + if err != nil { + return nil, nil, err + } + } + + blob, err := arch.SaveJSON(backend.Tree, &Tree{node}) + if err != nil { + return nil, nil, err + } + + sn.Content = blob.ID + + // save snapshot + sn.StorageMap = arch.smap + blob, err = arch.SaveJSON(backend.Snapshot, sn) + if err != nil { + return nil, nil, err + } + + return sn, blob, nil +} diff --git a/backend/doc.go b/backend/doc.go new file mode 100644 index 000000000..e97cb0e7f --- /dev/null +++ b/backend/doc.go @@ -0,0 +1,2 @@ +// Package backend provides local and remote storage for khepri backups. +package backend diff --git a/backend/generic.go b/backend/generic.go new file mode 100644 index 000000000..237712ced --- /dev/null +++ b/backend/generic.go @@ -0,0 +1,84 @@ +package backend + +import ( + "bytes" + "compress/zlib" + "crypto/sha256" + "io/ioutil" +) + +// Each lists all entries of type t in the backend and calls function f() with +// the id and data. +func Each(be Server, t Type, f func(id ID, data []byte, err error)) error { + ids, err := be.List(t) + if err != nil { + return err + } + + for _, id := range ids { + data, err := be.Get(t, id) + if err != nil { + f(id, nil, err) + continue + } + + f(id, data, nil) + } + + return nil +} + +// Each lists all entries of type t in the backend and calls function f() with +// the id. +func EachID(be Server, t Type, f func(ID)) error { + ids, err := be.List(t) + if err != nil { + return err + } + + for _, id := range ids { + f(id) + } + + return nil +} + +// Compress applies zlib compression to data. +func Compress(data []byte) []byte { + // apply zlib compression + var b bytes.Buffer + w := zlib.NewWriter(&b) + _, err := w.Write(data) + if err != nil { + panic(err) + } + w.Close() + + return b.Bytes() +} + +// Uncompress reverses zlib compression on data. +func Uncompress(data []byte) []byte { + b := bytes.NewBuffer(data) + r, err := zlib.NewReader(b) + if err != nil { + panic(err) + } + + buf, err := ioutil.ReadAll(r) + if err != nil { + panic(err) + } + + r.Close() + + return buf +} + +// Hash returns the ID for data. +func Hash(data []byte) ID { + h := sha256.Sum256(data) + id := make(ID, 32) + copy(id, h[:]) + return id +} diff --git a/backend/generic_test.go b/backend/generic_test.go new file mode 100644 index 000000000..5b8b968ca --- /dev/null +++ b/backend/generic_test.go @@ -0,0 +1,36 @@ +package backend_test + +import ( + "fmt" + "path/filepath" + "reflect" + "runtime" + "testing" +) + +// assert fails the test if the condition is false. +func assert(tb testing.TB, condition bool, msg string, v ...interface{}) { + if !condition { + _, file, line, _ := runtime.Caller(1) + fmt.Printf("\033[31m%s:%d: "+msg+"\033[39m\n\n", append([]interface{}{filepath.Base(file), line}, v...)...) + tb.FailNow() + } +} + +// ok fails the test if an err is not nil. +func ok(tb testing.TB, err error) { + if err != nil { + _, file, line, _ := runtime.Caller(1) + fmt.Printf("\033[31m%s:%d: unexpected error: %s\033[39m\n\n", filepath.Base(file), line, err.Error()) + tb.FailNow() + } +} + +// equals fails the test if exp is not equal to act. +func equals(tb testing.TB, exp, act interface{}) { + if !reflect.DeepEqual(exp, act) { + _, file, line, _ := runtime.Caller(1) + fmt.Printf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act) + tb.FailNow() + } +} diff --git a/id.go b/backend/id.go similarity index 66% rename from id.go rename to backend/id.go index b0ad178aa..ef9067040 100644 --- a/id.go +++ b/backend/id.go @@ -1,12 +1,15 @@ -package khepri +package backend import ( "bytes" "crypto/sha256" "encoding/hex" "encoding/json" + "errors" ) +const sha256_length = 32 // in bytes + // References content within a repository. type ID []byte @@ -18,6 +21,10 @@ func ParseID(s string) (ID, error) { return nil, err } + if len(b) != sha256_length { + return nil, errors.New("invalid length for sha256 hash") + } + return ID(b), nil } @@ -62,7 +69,37 @@ func (id *ID) UnmarshalJSON(b []byte) error { func IDFromData(d []byte) ID { hash := sha256.Sum256(d) - id := make([]byte, 32) + id := make([]byte, sha256_length) copy(id, hash[:]) return id } + +type IDs []ID + +func (ids IDs) Len() int { + return len(ids) +} + +func (ids IDs) Less(i, j int) bool { + if len(ids[i]) < len(ids[j]) { + return true + } + + for k, b := range ids[i] { + if b == ids[j][k] { + continue + } + + if b < ids[j][k] { + return true + } else { + return false + } + } + + return false +} + +func (ids IDs) Swap(i, j int) { + ids[i], ids[j] = ids[j], ids[i] +} diff --git a/backend/interface.go b/backend/interface.go new file mode 100644 index 000000000..28d42d04a --- /dev/null +++ b/backend/interface.go @@ -0,0 +1,21 @@ +package backend + +type Type string + +const ( + Blob Type = "blob" + Key = "key" + Lock = "lock" + Snapshot = "snapshot" + Tree = "tree" +) + +type Server interface { + Create(Type, []byte) (ID, error) + Get(Type, ID) ([]byte, error) + List(Type) (IDs, error) + Test(Type, ID) (bool, error) + Remove(Type, ID) error + + Location() string +} diff --git a/backend/local.go b/backend/local.go new file mode 100644 index 000000000..3e7d76458 --- /dev/null +++ b/backend/local.go @@ -0,0 +1,213 @@ +package backend + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" +) + +const ( + dirMode = 0700 + blobPath = "blobs" + snapshotPath = "snapshots" + treePath = "trees" + lockPath = "locks" + keyPath = "keys" + tempPath = "tmp" +) + +type Local struct { + p string +} + +// OpenLocal opens the local backend at dir. +func OpenLocal(dir string) (*Local, error) { + items := []string{ + dir, + filepath.Join(dir, blobPath), + filepath.Join(dir, snapshotPath), + filepath.Join(dir, treePath), + filepath.Join(dir, lockPath), + filepath.Join(dir, keyPath), + filepath.Join(dir, tempPath), + } + + // test if all necessary dirs and files are there + for _, d := range items { + if _, err := os.Stat(d); err != nil { + return nil, fmt.Errorf("%s does not exist", d) + } + } + + return &Local{p: dir}, nil +} + +// CreateLocal creates all the necessary files and directories for a new local +// backend at dir. +func CreateLocal(dir string) (*Local, error) { + dirs := []string{ + dir, + filepath.Join(dir, blobPath), + filepath.Join(dir, snapshotPath), + filepath.Join(dir, treePath), + filepath.Join(dir, lockPath), + filepath.Join(dir, keyPath), + filepath.Join(dir, tempPath), + } + + // test if directories already exist + for _, d := range dirs[1:] { + if _, err := os.Stat(d); err == nil { + return nil, fmt.Errorf("dir %s already exists", d) + } + } + + // create paths for blobs, refs and temp + for _, d := range dirs { + err := os.MkdirAll(d, dirMode) + if err != nil { + return nil, err + } + } + + // open repository + return OpenLocal(dir) +} + +// Location returns this backend's location (the directory name). +func (b *Local) Location() string { + return b.p +} + +// Return temp directory in correct directory for this backend. +func (b *Local) tempFile() (*os.File, error) { + return ioutil.TempFile(filepath.Join(b.p, tempPath), "temp-") +} + +// Rename temp file to final name according to type and ID. +func (b *Local) renameFile(file *os.File, t Type, id ID) error { + filename := filepath.Join(b.dir(t), id.String()) + return os.Rename(file.Name(), filename) +} + +// Construct directory for given Type. +func (b *Local) dir(t Type) string { + var n string + switch t { + case Blob: + n = blobPath + case Snapshot: + n = snapshotPath + case Tree: + n = treePath + case Lock: + n = lockPath + case Key: + n = keyPath + } + return filepath.Join(b.p, n) +} + +// Create stores new content of type t and data and returns the ID. +func (b *Local) Create(t Type, data []byte) (ID, error) { + // TODO: make sure that tempfile is removed upon error + + // create tempfile in repository + var err error + file, err := b.tempFile() + if err != nil { + return nil, err + } + + // write data to tempfile + _, err = file.Write(data) + if err != nil { + return nil, err + } + + // close tempfile, return id + id := IDFromData(data) + err = b.renameFile(file, t, id) + if err != nil { + return nil, err + } + + return id, nil +} + +// Construct path for given Type and ID. +func (b *Local) filename(t Type, id ID) string { + return filepath.Join(b.dir(t), id.String()) +} + +// Get returns the content stored under the given ID. +func (b *Local) Get(t Type, id ID) ([]byte, error) { + // try to open file + file, err := os.Open(b.filename(t, id)) + defer file.Close() + if err != nil { + return nil, err + } + + // read all + buf, err := ioutil.ReadAll(file) + if err != nil { + return nil, err + } + + return buf, nil +} + +// Test returns true if a blob of the given type and ID exists in the backend. +func (b *Local) Test(t Type, id ID) (bool, error) { + // try to open file + file, err := os.Open(b.filename(t, id)) + defer func() { + file.Close() + }() + + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + + return true, nil +} + +// Remove removes the content stored at ID. +func (b *Local) Remove(t Type, id ID) error { + return os.Remove(b.filename(t, id)) +} + +// List lists all objects of a given type. +func (b *Local) List(t Type) (IDs, error) { + // TODO: use os.Open() and d.Readdirnames() instead of Glob() + pattern := filepath.Join(b.dir(t), "*") + + matches, err := filepath.Glob(pattern) + if err != nil { + return nil, err + } + + ids := make(IDs, 0, len(matches)) + + for _, m := range matches { + base := filepath.Base(m) + + if base == "" { + continue + } + id, err := ParseID(base) + + if err != nil { + continue + } + + ids = append(ids, id) + } + + return ids, nil +} diff --git a/backend/local_test.go b/backend/local_test.go new file mode 100644 index 000000000..938506890 --- /dev/null +++ b/backend/local_test.go @@ -0,0 +1,171 @@ +package backend_test + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + "sort" + "testing" + + "github.com/fd0/khepri/backend" +) + +var testCleanup = flag.Bool("test.cleanup", true, "clean up after running tests (remove local backend directory with all content)") + +var TestStrings = []struct { + id string + data string +}{ + {"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", "foobar"}, + {"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, + {"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", "foo/bar"}, + {"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", "foo/../../baz"}, +} + +func setupBackend(t *testing.T) *backend.Local { + tempdir, err := ioutil.TempDir("", "khepri-test-") + ok(t, err) + + b, err := backend.CreateLocal(tempdir) + ok(t, err) + + t.Logf("created local backend at %s", tempdir) + + return b +} + +func teardownBackend(t *testing.T, b *backend.Local) { + if !*testCleanup { + t.Logf("leaving local backend at %s\n", b.Location()) + return + } + + ok(t, os.RemoveAll(b.Location())) +} + +func testBackend(b backend.Server, t *testing.T) { + for _, tpe := range []backend.Type{backend.Blob, backend.Key, backend.Lock, backend.Snapshot, backend.Tree} { + // detect non-existing files + for _, test := range TestStrings { + id, err := backend.ParseID(test.id) + ok(t, err) + + // test if blob is already in repository + ret, err := b.Test(tpe, id) + ok(t, err) + assert(t, !ret, "blob was found to exist before creating") + + // try to open not existing blob + d, err := b.Get(tpe, id) + assert(t, err != nil && d == nil, "blob data could be extracted befor creation") + + // try to get string out, should fail + ret, err = b.Test(tpe, id) + ok(t, err) + assert(t, !ret, fmt.Sprintf("id %q was found (but should not have)", test.id)) + } + + // add files + for _, test := range TestStrings { + // store string in backend + id, err := b.Create(tpe, []byte(test.data)) + ok(t, err) + + equals(t, test.id, id.String()) + + // try to get it out again + buf, err := b.Get(tpe, id) + ok(t, err) + assert(t, buf != nil, "Get() returned nil") + + // compare content + equals(t, test.data, string(buf)) + } + + // list items + IDs := backend.IDs{} + + for _, test := range TestStrings { + id, err := backend.ParseID(test.id) + ok(t, err) + IDs = append(IDs, id) + } + + ids, err := b.List(tpe) + ok(t, err) + + sort.Sort(ids) + sort.Sort(IDs) + equals(t, IDs, ids) + + // remove content if requested + if *testCleanup { + for _, test := range TestStrings { + id, err := backend.ParseID(test.id) + ok(t, err) + + found, err := b.Test(tpe, id) + ok(t, err) + assert(t, found, fmt.Sprintf("id %q was not found before removal")) + + ok(t, b.Remove(tpe, id)) + + found, err = b.Test(tpe, id) + ok(t, err) + assert(t, !found, fmt.Sprintf("id %q was not found before removal")) + } + } + + } +} + +func TestBackend(t *testing.T) { + // test for non-existing backend + b, err := backend.OpenLocal("/invalid-khepri-test") + assert(t, err != nil, "opening invalid repository at /invalid-khepri-test should have failed, but err is nil") + assert(t, b == nil, fmt.Sprintf("opening invalid repository at /invalid-khepri-test should have failed, but b is not nil: %v", b)) + + b = setupBackend(t) + defer teardownBackend(t, b) + + testBackend(b, t) +} + +func TestLocalBackendCreationFailures(t *testing.T) { + b := setupBackend(t) + defer teardownBackend(t, b) + + // test failure to create a new repository at the same location + b2, err := backend.CreateLocal(b.Location()) + assert(t, err != nil && b2 == nil, fmt.Sprintf("creating a repository at %s for the second time should have failed", b.Location())) + + // test failure to create a new repository at the same location without a config file + b2, err = backend.CreateLocal(b.Location()) + assert(t, err != nil && b2 == nil, fmt.Sprintf("creating a repository at %s for the second time should have failed", b.Location())) +} + +func TestID(t *testing.T) { + for _, test := range TestStrings { + id, err := backend.ParseID(test.id) + ok(t, err) + + id2, err := backend.ParseID(test.id) + ok(t, err) + assert(t, id.Equal(id2), "ID.Equal() does not work as expected") + + ret, err := id.EqualString(test.id) + ok(t, err) + assert(t, ret, "ID.EqualString() returned wrong value") + + // test json marshalling + buf, err := id.MarshalJSON() + ok(t, err) + equals(t, "\""+test.id+"\"", string(buf)) + + var id3 backend.ID + err = id3.UnmarshalJSON(buf) + ok(t, err) + equals(t, id, id3) + } +} diff --git a/cmd/archive/main.go b/cmd/archive/main.go new file mode 100644 index 000000000..2b17f085f --- /dev/null +++ b/cmd/archive/main.go @@ -0,0 +1,81 @@ +package main + +import ( + "fmt" + "os" + + "github.com/fd0/khepri" + "github.com/fd0/khepri/backend" +) + +const pass = "foobar" + +func main() { + if len(os.Args) != 3 { + fmt.Fprintf(os.Stderr, "usage: archive REPO DIR\n") + os.Exit(1) + } + repo := os.Args[1] + dir := os.Args[2] + + // fmt.Printf("import %s into backend %s\n", dir, repo) + + var ( + be backend.Server + key *khepri.Key + ) + + be, err := backend.OpenLocal(repo) + if err != nil { + fmt.Printf("creating %s\n", repo) + be, err = backend.CreateLocal(repo) + if err != nil { + fmt.Fprintf(os.Stderr, "failed: %v\n", err) + os.Exit(2) + } + + key, err = khepri.CreateKey(be, pass) + if err != nil { + fmt.Fprintf(os.Stderr, "failed: %v\n", err) + os.Exit(2) + } + } + + key, err = khepri.SearchKey(be, pass) + if err != nil { + fmt.Fprintf(os.Stderr, "failed: %v\n", err) + os.Exit(2) + } + + arch, err := khepri.NewArchiver(be, key) + if err != nil { + fmt.Fprintf(os.Stderr, "err: %v\n", err) + } + arch.Error = func(dir string, fi os.FileInfo, err error) error { + fmt.Fprintf(os.Stderr, "error for %s: %v\n%s\n", dir, err, fi) + return err + } + + arch.Filter = func(item string, fi os.FileInfo) bool { + // if fi.IsDir() { + // if fi.Name() == ".svn" { + // return false + // } + // } else { + // if filepath.Ext(fi.Name()) == ".bz2" { + // return false + // } + // } + + fmt.Printf("%s\n", item) + return true + } + + _, blob, err := arch.Import(dir) + if err != nil { + fmt.Fprintf(os.Stderr, "Import() error: %v\n", err) + os.Exit(2) + } + + fmt.Printf("saved as %+v\n", blob) +} diff --git a/cmd/cat/main.go b/cmd/cat/main.go new file mode 100644 index 000000000..2185736cc --- /dev/null +++ b/cmd/cat/main.go @@ -0,0 +1,126 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "time" + + "code.google.com/p/go.crypto/ssh/terminal" + + "github.com/fd0/khepri" + "github.com/fd0/khepri/backend" +) + +func read_password(prompt string) string { + p := os.Getenv("KHEPRI_PASSWORD") + if p != "" { + return p + } + + fmt.Print(prompt) + pw, err := terminal.ReadPassword(int(os.Stdin.Fd())) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to read password: %v", err) + os.Exit(2) + } + fmt.Println() + + return string(pw) +} + +func json_pp(data []byte) error { + var buf bytes.Buffer + err := json.Indent(&buf, data, "", " ") + if err != nil { + return err + } + + fmt.Println(string(buf.Bytes())) + return nil +} + +type StopWatch struct { + start, last time.Time +} + +func NewStopWatch() *StopWatch { + return &StopWatch{ + start: time.Now(), + last: time.Now(), + } +} + +func (s *StopWatch) Next(format string, data ...interface{}) { + t := time.Now() + d := t.Sub(s.last) + s.last = t + arg := make([]interface{}, len(data)+1) + arg[0] = d + copy(arg[1:], data) + fmt.Printf("[%s]: "+format+"\n", arg...) +} + +func main() { + if len(os.Args) != 3 { + fmt.Fprintf(os.Stderr, "usage: cat REPO ID\n") + os.Exit(1) + } + repo := os.Args[1] + id, err := backend.ParseID(filepath.Base(os.Args[2])) + if err != nil { + panic(err) + } + + s := NewStopWatch() + + be, err := backend.OpenLocal(repo) + if err != nil { + fmt.Fprintf(os.Stderr, "failed: %v\n", err) + os.Exit(1) + } + + s.Next("OpenLocal()") + + key, err := khepri.SearchKey(be, read_password("Enter Password for Repository: ")) + if err != nil { + fmt.Fprintf(os.Stderr, "failed: %v\n", err) + os.Exit(2) + } + + s.Next("SearchKey()") + + // try all possible types + for _, t := range []backend.Type{backend.Blob, backend.Snapshot, backend.Lock, backend.Tree, backend.Key} { + buf, err := be.Get(t, id) + if err != nil { + continue + } + + s.Next("Get(%s, %s)", t, id) + + if t == backend.Key { + json_pp(buf) + } + + buf2, err := key.Decrypt(buf) + if err != nil { + panic(err) + } + + if t == backend.Blob { + // directly output blob + fmt.Println(string(buf2)) + } else { + // try to uncompress and print as idented json + err = json_pp(backend.Uncompress(buf2)) + if err != nil { + fmt.Fprintf(os.Stderr, "failed: %v\n", err) + } + } + + break + } +} diff --git a/cmd/decrypt/main.go b/cmd/decrypt/main.go new file mode 100644 index 000000000..da40cfe68 --- /dev/null +++ b/cmd/decrypt/main.go @@ -0,0 +1,233 @@ +package main + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + + "crypto/aes" + "crypto/cipher" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + + "code.google.com/p/go.crypto/scrypt" + "code.google.com/p/go.crypto/ssh/terminal" + + "github.com/jessevdk/go-flags" +) + +const ( + scrypt_N = 65536 + scrypt_r = 8 + scrypt_p = 1 + aesKeySize = 32 // for AES256 +) + +var Opts struct { + Password string `short:"p" long:"password" description:"Password for the file"` + Keys string `short:"k" long:"keys" description:"Keys for the file (encryption_key || sign_key, hex-encoded)"` + Salt string `short:"s" long:"salt" description:"Salt to use (hex-encoded)"` +} + +func newIV() ([]byte, error) { + buf := make([]byte, aes.BlockSize) + _, err := io.ReadFull(rand.Reader, buf) + if err != nil { + return nil, err + } + + return buf, nil +} + +func pad(plaintext []byte) []byte { + l := aes.BlockSize - (len(plaintext) % aes.BlockSize) + if l == 0 { + l = aes.BlockSize + } + + if l <= 0 || l > aes.BlockSize { + panic("invalid padding size") + } + + return append(plaintext, bytes.Repeat([]byte{byte(l)}, l)...) +} + +func unpad(plaintext []byte) []byte { + l := len(plaintext) + pad := plaintext[l-1] + + if pad > aes.BlockSize { + panic(errors.New("padding > BlockSize")) + } + + if pad == 0 { + panic(errors.New("invalid padding 0")) + } + + for i := l - int(pad); i < l; i++ { + if plaintext[i] != pad { + panic(errors.New("invalid padding!")) + } + } + + return plaintext[:l-int(pad)] +} + +// Encrypt encrypts and signs data. Returned is IV || Ciphertext || HMAC. For +// the hash function, SHA256 is used, so the overhead is 16+32=48 byte. +func Encrypt(ekey, skey []byte, plaintext []byte) ([]byte, error) { + iv, err := newIV() + if err != nil { + panic(fmt.Sprintf("unable to generate new random iv: %v", err)) + } + + c, err := aes.NewCipher(ekey) + if err != nil { + panic(fmt.Sprintf("unable to create cipher: %v", err)) + } + + e := cipher.NewCBCEncrypter(c, iv) + p := pad(plaintext) + ciphertext := make([]byte, len(p)) + e.CryptBlocks(ciphertext, p) + + ciphertext = append(iv, ciphertext...) + + hm := hmac.New(sha256.New, skey) + + n, err := hm.Write(ciphertext) + if err != nil || n != len(ciphertext) { + panic(fmt.Sprintf("unable to calculate hmac of ciphertext: %v", err)) + } + + return hm.Sum(ciphertext), nil +} + +// Decrypt verifes and decrypts the ciphertext. Ciphertext must be in the form +// IV || Ciphertext || HMAC. +func Decrypt(ekey, skey []byte, ciphertext []byte) ([]byte, error) { + hm := hmac.New(sha256.New, skey) + + // extract hmac + l := len(ciphertext) - hm.Size() + ciphertext, mac := ciphertext[:l], ciphertext[l:] + + // calculate new hmac + n, err := hm.Write(ciphertext) + if err != nil || n != len(ciphertext) { + panic(fmt.Sprintf("unable to calculate hmac of ciphertext, err %v", err)) + } + + // verify hmac + mac2 := hm.Sum(nil) + if !hmac.Equal(mac, mac2) { + panic("HMAC verification failed") + } + + // extract iv + iv, ciphertext := ciphertext[:aes.BlockSize], ciphertext[aes.BlockSize:] + + // decrypt data + c, err := aes.NewCipher(ekey) + if err != nil { + panic(fmt.Sprintf("unable to create cipher: %v", err)) + } + + // decrypt + e := cipher.NewCBCDecrypter(c, iv) + plaintext := make([]byte, len(ciphertext)) + e.CryptBlocks(plaintext, ciphertext) + + // remove padding and return + return unpad(plaintext), nil +} + +func errx(code int, format string, data ...interface{}) { + if len(format) > 0 && format[len(format)-1] != '\n' { + format += "\n" + } + fmt.Fprintf(os.Stderr, format, data...) + os.Exit(code) +} + +func read_password(prompt string) string { + p := os.Getenv("KHEPRI_PASSWORD") + if p != "" { + return p + } + + fmt.Print(prompt) + pw, err := terminal.ReadPassword(int(os.Stdin.Fd())) + if err != nil { + errx(2, "unable to read password: %v", err) + } + fmt.Println() + + return string(pw) +} + +func main() { + args, err := flags.Parse(&Opts) + if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp { + os.Exit(0) + } + + var keys []byte + + if Opts.Password == "" && Opts.Keys == "" { + Opts.Password = read_password("password: ") + + salt, err := hex.DecodeString(Opts.Salt) + if err != nil { + errx(1, "unable to hex-decode salt: %v", err) + } + + keys, err = scrypt.Key([]byte(Opts.Password), salt, scrypt_N, scrypt_r, scrypt_p, 2*aesKeySize) + if err != nil { + errx(1, "scrypt: %v", err) + } + } + + if Opts.Keys != "" { + keys, err = hex.DecodeString(Opts.Keys) + if err != nil { + errx(1, "unable to hex-decode keys: %v", err) + } + } + + if len(keys) != 2*aesKeySize { + errx(2, "key length is not 512") + } + + encrypt_key := keys[:aesKeySize] + sign_key := keys[aesKeySize:] + + for _, filename := range args { + f, err := os.Open(filename) + defer f.Close() + if err != nil { + errx(3, "%v\n", err) + } + + buf, err := ioutil.ReadAll(f) + if err != nil { + errx(3, "%v\n", err) + } + + buf, err = Decrypt(encrypt_key, sign_key, buf) + if err != nil { + errx(3, "%v\n", err) + } + + _, err = os.Stdout.Write(buf) + if err != nil { + errx(3, "%v\n", err) + } + } + +} diff --git a/cmd/dirdiff/main.go b/cmd/dirdiff/main.go index a1ffd37bc..7af0981bb 100644 --- a/cmd/dirdiff/main.go +++ b/cmd/dirdiff/main.go @@ -53,14 +53,17 @@ func walk(dir string) <-chan *entry { func (e *entry) equals(other *entry) bool { if e.path != other.path { + fmt.Printf("path does not match\n") return false } if e.fi.Mode() != other.fi.Mode() { + fmt.Printf("mode does not match\n") return false } if e.fi.ModTime() != other.fi.ModTime() { + fmt.Printf("ModTime does not match\n") return false } diff --git a/cmd/khepri/cmd_backup.go b/cmd/khepri/cmd_backup.go index 5a3c2f277..97fe80b0b 100644 --- a/cmd/khepri/cmd_backup.go +++ b/cmd/khepri/cmd_backup.go @@ -3,37 +3,34 @@ package main import ( "errors" "fmt" - "log" + "os" "github.com/fd0/khepri" + "github.com/fd0/khepri/backend" ) -func commandBackup(repo *khepri.Repository, args []string) error { +func commandBackup(be backend.Server, key *khepri.Key, args []string) error { if len(args) != 1 { - return errors.New("usage: backup dir") + return errors.New("usage: backup [dir|file]") } target := args[0] - tree, err := khepri.NewTreeFromPath(repo, target) + arch, err := khepri.NewArchiver(be, key) + if err != nil { + fmt.Fprintf(os.Stderr, "err: %v\n", err) + } + arch.Error = func(dir string, fi os.FileInfo, err error) error { + fmt.Fprintf(os.Stderr, "error for %s: %v\n%s\n", dir, err, fi) + return err + } + + _, blob, err := arch.Import(target) if err != nil { return err } - id, err := tree.Save(repo) - if err != nil { - return err - } - - sn := khepri.NewSnapshot(target) - sn.Content = id - snid, err := sn.Save(repo) - - if err != nil { - log.Printf("error saving snapshopt: %v", err) - } - - fmt.Printf("%q archived as %v\n", target, snid) + fmt.Printf("snapshot %s saved\n", blob.Storage) return nil } diff --git a/cmd/khepri/cmd_dump.go b/cmd/khepri/cmd_dump.go deleted file mode 100644 index d0fb09371..000000000 --- a/cmd/khepri/cmd_dump.go +++ /dev/null @@ -1,90 +0,0 @@ -package main - -import ( - "encoding/json" - "errors" - "fmt" - "io" - "log" - "os" - - "github.com/fd0/khepri" -) - -func dump_tree(repo *khepri.Repository, id khepri.ID) error { - tree, err := khepri.NewTreeFromRepo(repo, id) - if err != nil { - return err - } - - buf, err := json.MarshalIndent(tree, "", " ") - if err != nil { - return err - } - - fmt.Printf("tree %s\n%s\n", id, buf) - - for _, node := range tree.Nodes { - if node.Type == "dir" { - err = dump_tree(repo, node.Subtree) - if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - } - } - } - - return nil -} - -func dump_snapshot(repo *khepri.Repository, id khepri.ID) error { - sn, err := khepri.LoadSnapshot(repo, id) - if err != nil { - log.Fatalf("error loading snapshot %s", id) - } - - buf, err := json.MarshalIndent(sn, "", " ") - if err != nil { - return err - } - - fmt.Printf("%s\n%s\n", sn, buf) - - return dump_tree(repo, sn.Content) -} - -func dump_file(repo *khepri.Repository, id khepri.ID) error { - rd, err := repo.Get(khepri.TYPE_BLOB, id) - if err != nil { - return err - } - - io.Copy(os.Stdout, rd) - - return nil -} - -func commandDump(repo *khepri.Repository, args []string) error { - if len(args) != 2 { - return errors.New("usage: dump [snapshot|tree|file] ID") - } - - tpe := args[0] - - id, err := khepri.ParseID(args[1]) - if err != nil { - errx(1, "invalid id %q: %v", args[0], err) - } - - switch tpe { - case "snapshot": - return dump_snapshot(repo, id) - case "tree": - return dump_tree(repo, id) - case "file": - return dump_file(repo, id) - default: - return fmt.Errorf("invalid type %q", tpe) - } - - return nil -} diff --git a/cmd/khepri/cmd_fsck.go b/cmd/khepri/cmd_fsck.go index 2b7658f88..63ea29680 100644 --- a/cmd/khepri/cmd_fsck.go +++ b/cmd/khepri/cmd_fsck.go @@ -1,84 +1,76 @@ package main -import ( - "encoding/json" - "io/ioutil" - "log" +import "github.com/fd0/khepri/backend" - "github.com/fd0/khepri" -) +// func fsck_tree(be backend.Server, id backend.ID) (bool, error) { +// log.Printf(" checking dir %s", id) -func fsck_tree(repo *khepri.Repository, id khepri.ID) (bool, error) { - log.Printf(" checking dir %s", id) +// buf, err := be.GetBlob(id) +// if err != nil { +// return false, err +// } - rd, err := repo.Get(khepri.TYPE_BLOB, id) - if err != nil { - return false, err - } +// tree := &khepri.Tree{} +// err = json.Unmarshal(buf, tree) +// if err != nil { +// return false, err +// } - buf, err := ioutil.ReadAll(rd) +// if !id.Equal(backend.IDFromData(buf)) { +// return false, nil +// } - tree := &khepri.Tree{} - err = json.Unmarshal(buf, tree) - if err != nil { - return false, err - } +// return true, nil +// } - if !id.Equal(khepri.IDFromData(buf)) { - return false, nil - } +// func fsck_snapshot(be backend.Server, id backend.ID) (bool, error) { +// log.Printf("checking snapshot %s", id) - return true, nil -} +// sn, err := khepri.LoadSnapshot(be, id) +// if err != nil { +// return false, err +// } -func fsck_snapshot(repo *khepri.Repository, id khepri.ID) (bool, error) { - log.Printf("checking snapshot %s", id) +// return fsck_tree(be, sn.Content) +// } - sn, err := khepri.LoadSnapshot(repo, id) - if err != nil { - return false, err - } +func commandFsck(be backend.Server, args []string) error { + // var snapshots backend.IDs + // var err error - return fsck_tree(repo, sn.Content) -} + // if len(args) != 0 { + // snapshots = make(backend.IDs, 0, len(args)) -func commandFsck(repo *khepri.Repository, args []string) error { - var snapshots khepri.IDs - var err error + // for _, arg := range args { + // id, err := backend.ParseID(arg) + // if err != nil { + // log.Fatal(err) + // } - if len(args) != 0 { - snapshots = make(khepri.IDs, 0, len(args)) + // snapshots = append(snapshots, id) + // } + // } else { + // snapshots, err = be.ListRefs() - for _, arg := range args { - id, err := khepri.ParseID(arg) - if err != nil { - log.Fatal(err) - } + // if err != nil { + // log.Fatalf("error reading list of snapshot IDs: %v", err) + // } + // } - snapshots = append(snapshots, id) - } - } else { - snapshots, err = repo.List(khepri.TYPE_REF) + // log.Printf("checking %d snapshots", len(snapshots)) - if err != nil { - log.Fatalf("error reading list of snapshot IDs: %v", err) - } - } + // for _, id := range snapshots { + // ok, err := fsck_snapshot(be, id) - log.Printf("checking %d snapshots", len(snapshots)) + // if err != nil { + // log.Printf("error checking snapshot %s: %v", id, err) + // continue + // } - for _, id := range snapshots { - ok, err := fsck_snapshot(repo, id) - - if err != nil { - log.Printf("error checking snapshot %s: %v", id, err) - continue - } - - if !ok { - log.Printf("snapshot %s failed", id) - } - } + // if !ok { + // log.Printf("snapshot %s failed", id) + // } + // } return nil } diff --git a/cmd/khepri/cmd_init.go b/cmd/khepri/cmd_init.go index cd64d8dfd..b7992bcd7 100644 --- a/cmd/khepri/cmd_init.go +++ b/cmd/khepri/cmd_init.go @@ -5,16 +5,30 @@ import ( "os" "github.com/fd0/khepri" + "github.com/fd0/khepri/backend" ) func commandInit(path string) error { - repo, err := khepri.CreateRepository(path) + pw := read_password("enter password for new backend: ") + pw2 := read_password("enter password again: ") + + if pw != pw2 { + errx(1, "passwords do not match") + } + + be, err := backend.CreateLocal(path) if err != nil { - fmt.Fprintf(os.Stderr, "creating repository at %s failed: %v\n", path, err) + fmt.Fprintf(os.Stderr, "creating local backend at %s failed: %v\n", path, err) os.Exit(1) } - fmt.Printf("created khepri repository at %s\n", repo.Path()) + _, err = khepri.CreateKey(be, pw) + if err != nil { + fmt.Fprintf(os.Stderr, "creating key in local backend at %s failed: %v\n", path, err) + os.Exit(1) + } + + fmt.Printf("created khepri backend at %s\n", be.Location()) return nil } diff --git a/cmd/khepri/cmd_list.go b/cmd/khepri/cmd_list.go index fc14f5e16..f49cdbf96 100644 --- a/cmd/khepri/cmd_list.go +++ b/cmd/khepri/cmd_list.go @@ -1,29 +1,21 @@ package main import ( - "errors" - "fmt" - "os" - "github.com/fd0/khepri" + "github.com/fd0/khepri/backend" ) -func commandList(repo *khepri.Repository, args []string) error { - if len(args) != 1 { - return errors.New("usage: list [blob|ref]") - } +func commandList(be backend.Server, key *khepri.Key, args []string) error { - tpe := khepri.NewTypeFromString(args[0]) + // ids, err := be.ListRefs() + // if err != nil { + // fmt.Fprintf(os.Stderr, "error: %v\n", err) + // return nil + // } - ids, err := repo.List(tpe) - if err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - return nil - } - - for _, id := range ids { - fmt.Printf("%v\n", id) - } + // for _, id := range ids { + // fmt.Printf("%v\n", id) + // } return nil } diff --git a/cmd/khepri/cmd_restore.go b/cmd/khepri/cmd_restore.go index 608542257..4672b2cce 100644 --- a/cmd/khepri/cmd_restore.go +++ b/cmd/khepri/cmd_restore.go @@ -2,34 +2,54 @@ package main import ( "errors" - "log" + "fmt" + "os" "github.com/fd0/khepri" + "github.com/fd0/khepri/backend" ) -func commandRestore(repo *khepri.Repository, args []string) error { +func commandRestore(be backend.Server, key *khepri.Key, args []string) error { if len(args) != 2 { return errors.New("usage: restore ID dir") } - id, err := khepri.ParseID(args[0]) + id, err := backend.ParseID(args[0]) if err != nil { errx(1, "invalid id %q: %v", args[0], err) } target := args[1] - sn, err := khepri.LoadSnapshot(repo, id) + // create restorer + res, err := khepri.NewRestorer(be, key, id) if err != nil { - log.Fatalf("error loading snapshot %s: %v", id, err) + fmt.Fprintf(os.Stderr, "creating restorer failed: %v\n", err) + os.Exit(2) } - err = sn.RestoreAt(target) - if err != nil { - log.Fatalf("error restoring snapshot %s: %v", id, err) + res.Error = func(dir string, node *khepri.Node, err error) error { + fmt.Fprintf(os.Stderr, "error for %s: %+v\n", dir, err) + + // if node.Type == "dir" { + // if e, ok := err.(*os.PathError); ok { + // if errn, ok := e.Err.(syscall.Errno); ok { + // if errn == syscall.EEXIST { + // fmt.Printf("ignoring already existing directory %s\n", dir) + // return nil + // } + // } + // } + // } + return err } - log.Printf("%q restored to %q\n", id, target) + fmt.Printf("restoring %s to %s\n", res.Snapshot(), target) + + err = res.RestoreTo(target) + if err != nil { + return err + } return nil } diff --git a/cmd/khepri/cmd_snapshots.go b/cmd/khepri/cmd_snapshots.go index cf9523e6b..84af0dbe9 100644 --- a/cmd/khepri/cmd_snapshots.go +++ b/cmd/khepri/cmd_snapshots.go @@ -3,39 +3,34 @@ package main import ( "errors" "fmt" - "log" "github.com/fd0/khepri" + "github.com/fd0/khepri/backend" ) const TimeFormat = "02.01.2006 15:04:05 -0700" -func commandSnapshots(repo *khepri.Repository, args []string) error { +func commandSnapshots(be backend.Server, key *khepri.Key, args []string) error { if len(args) != 0 { return errors.New("usage: snapshots") } - snapshot_ids, err := repo.List(khepri.TYPE_REF) - if err != nil { - log.Fatalf("error loading list of snapshot ids: %v", err) - } + // ch, err := khepri.NewContentHandler(be, key) + // if err != nil { + // return err + // } - fmt.Printf("found snapshots:\n") - for _, id := range snapshot_ids { - snapshot, err := khepri.LoadSnapshot(repo, id) + backend.EachID(be, backend.Snapshot, func(id backend.ID) { + // sn, err := ch.LoadSnapshot(id) + // if err != nil { + // fmt.Fprintf(os.Stderr, "error loading snapshot %s: %v\n", id, err) + // return + // } - if err != nil { - log.Printf("error loading snapshot %s: %v", id, err) - continue - } - - fmt.Printf("%s %s@%s %s %s\n", - snapshot.Time.Format(TimeFormat), - snapshot.Username, - snapshot.Hostname, - snapshot.Dir, - id) - } + // fmt.Printf("snapshot %s\n %s at %s by %s\n", + // id, sn.Dir, sn.Time, sn.Username) + fmt.Println(id) + }) return nil } diff --git a/cmd/khepri/main.go b/cmd/khepri/main.go index 0a2daadf7..0ef5c4635 100644 --- a/cmd/khepri/main.go +++ b/cmd/khepri/main.go @@ -4,8 +4,13 @@ import ( "fmt" "log" "os" + "sort" + "strings" + + "code.google.com/p/go.crypto/ssh/terminal" "github.com/fd0/khepri" + "github.com/fd0/khepri/backend" "github.com/jessevdk/go-flags" ) @@ -21,18 +26,32 @@ func errx(code int, format string, data ...interface{}) { os.Exit(code) } -type commandFunc func(*khepri.Repository, []string) error +type commandFunc func(backend.Server, *khepri.Key, []string) error var commands map[string]commandFunc +func read_password(prompt string) string { + p := os.Getenv("KHEPRI_PASSWORD") + if p != "" { + return p + } + + fmt.Print(prompt) + pw, err := terminal.ReadPassword(int(os.Stdin.Fd())) + if err != nil { + errx(2, "unable to read password: %v", err) + } + fmt.Println() + + return string(pw) +} + func init() { commands = make(map[string]commandFunc) commands["backup"] = commandBackup commands["restore"] = commandRestore commands["list"] = commandList commands["snapshots"] = commandSnapshots - commands["fsck"] = commandFsck - commands["dump"] = commandDump } func main() { @@ -42,12 +61,22 @@ func main() { if Opts.Repo == "" { Opts.Repo = "khepri-backup" } - args, err := flags.Parse(&Opts) + args, err := flags.Parse(&Opts) if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp { os.Exit(0) } + if len(args) == 0 { + cmds := []string{"init"} + for k := range commands { + cmds = append(cmds, k) + } + sort.Strings(cmds) + fmt.Printf("nothing to do, available commands: [%v]\n", strings.Join(cmds, "|")) + os.Exit(0) + } + cmd := args[0] if cmd == "init" { @@ -64,13 +93,18 @@ func main() { errx(1, "unknown command: %q\n", cmd) } - repo, err := khepri.NewRepository(Opts.Repo) - + // read_password("enter password: ") + repo, err := backend.OpenLocal(Opts.Repo) if err != nil { errx(1, "unable to open repo: %v", err) } - err = f(repo, args[1:]) + key, err := khepri.SearchKey(repo, read_password("Enter Password for Repository: ")) + if err != nil { + errx(2, "unable to open repo: %v", err) + } + + err = f(repo, key, args[1:]) if err != nil { errx(1, "error executing command %q: %v", cmd, err) } diff --git a/cmd/list/main.go b/cmd/list/main.go new file mode 100644 index 000000000..884f59ac9 --- /dev/null +++ b/cmd/list/main.go @@ -0,0 +1,110 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + + "code.google.com/p/go.crypto/ssh/terminal" + + "github.com/fd0/khepri" + "github.com/fd0/khepri/backend" +) + +func read_password(prompt string) string { + p := os.Getenv("KHEPRI_PASSWORD") + if p != "" { + return p + } + + fmt.Print(prompt) + pw, err := terminal.ReadPassword(int(os.Stdin.Fd())) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to read password: %v", err) + os.Exit(2) + } + fmt.Println() + + return string(pw) +} + +func list(be backend.Server, key *khepri.Key, t backend.Type) { + ids, err := be.List(t) + if err != nil { + fmt.Fprintf(os.Stderr, "failed: %v\n", err) + os.Exit(3) + } + + for _, id := range ids { + buf, err := be.Get(t, id) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to get snapshot %s: %v\n", id, err) + continue + } + + if t != backend.Key && t != backend.Blob { + buf, err = key.Decrypt(buf) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + continue + } + + } + + if t == backend.Snapshot { + var sn khepri.Snapshot + err = json.Unmarshal(backend.Uncompress(buf), &sn) + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + continue + } + + fmt.Printf("%s %s\n", id, sn.String()) + } else if t == backend.Blob { + fmt.Printf("%s %d bytes (encrypted)\n", id, len(buf)) + } else if t == backend.Tree { + fmt.Printf("%s\n", backend.Hash(buf)) + } else if t == backend.Key { + k := &khepri.Key{} + err = json.Unmarshal(buf, k) + if err != nil { + fmt.Fprintf(os.Stderr, "unable to unmashal key: %v\n", err) + continue + } + fmt.Println(key) + } else if t == backend.Lock { + fmt.Printf("lock: %v\n", id) + } + } +} + +func main() { + if len(os.Args) != 2 { + fmt.Fprintf(os.Stderr, "usage: archive REPO\n") + os.Exit(1) + } + repo := os.Args[1] + + be, err := backend.OpenLocal(repo) + if err != nil { + fmt.Fprintf(os.Stderr, "failed: %v\n", err) + os.Exit(1) + } + + key, err := khepri.SearchKey(be, read_password("Enter Password for Repository: ")) + if err != nil { + fmt.Fprintf(os.Stderr, "failed: %v\n", err) + os.Exit(2) + } + + fmt.Printf("keys:\n") + list(be, key, backend.Key) + fmt.Printf("---\nlocks:\n") + list(be, key, backend.Lock) + fmt.Printf("---\nsnapshots:\n") + list(be, key, backend.Snapshot) + fmt.Printf("---\ntrees:\n") + list(be, key, backend.Tree) + fmt.Printf("---\nblobs:\n") + list(be, key, backend.Blob) +} diff --git a/contenthandler.go b/contenthandler.go new file mode 100644 index 000000000..47fecfe92 --- /dev/null +++ b/contenthandler.go @@ -0,0 +1,242 @@ +package khepri + +import ( + "encoding/json" + "errors" + "io" + "io/ioutil" + "os" + + "github.com/fd0/khepri/backend" + "github.com/fd0/khepri/chunker" +) + +type ContentHandler struct { + be backend.Server + key *Key + + content *StorageMap +} + +// NewContentHandler creates a new content handler. +func NewContentHandler(be backend.Server, key *Key) (*ContentHandler, error) { + ch := &ContentHandler{ + be: be, + key: key, + content: NewStorageMap(), + } + + return ch, nil +} + +// LoadSnapshotadds all blobs from a snapshot into the content handler and returns the snapshot. +func (ch *ContentHandler) LoadSnapshot(id backend.ID) (*Snapshot, error) { + sn, err := LoadSnapshot(ch, id) + if err != nil { + return nil, err + } + + ch.content.Merge(sn.StorageMap) + return sn, nil +} + +// LoadAllSnapshots adds all blobs from all snapshots that can be decrypted +// into the content handler. +func (ch *ContentHandler) LoadAllSnapshots() error { + // add all maps from all snapshots that can be decrypted to the storage map + err := backend.EachID(ch.be, backend.Snapshot, func(id backend.ID) { + sn, err := LoadSnapshot(ch, id) + if err != nil { + return + } + ch.content.Merge(sn.StorageMap) + }) + if err != nil { + return err + } + + return nil +} + +// Save encrypts data and stores it to the backend as type t. If the data was +// already saved before, the blob is returned. +func (ch *ContentHandler) Save(t backend.Type, data []byte) (*Blob, error) { + // compute plaintext hash + id := backend.Hash(data) + + // test if the hash is already in the backend + blob := ch.content.Find(id) + if blob != nil { + return blob, nil + } + + // else create a new blob + blob = &Blob{ + ID: id, + Size: uint64(len(data)), + } + + // encrypt blob + ciphertext, err := ch.key.Encrypt(data) + if err != nil { + return nil, err + } + + // save blob + sid, err := ch.be.Create(t, ciphertext) + if err != nil { + return nil, err + } + + blob.Storage = sid + blob.StorageSize = uint64(len(ciphertext)) + + // insert blob into the storage map + ch.content.Insert(blob) + + return blob, nil +} + +// SaveJSON serialises item as JSON and uses Save() to store it to the backend as type t. +func (ch *ContentHandler) SaveJSON(t backend.Type, item interface{}) (*Blob, error) { + // convert to json + data, err := json.Marshal(item) + if err != nil { + return nil, err + } + + // compress and save data + return ch.Save(t, backend.Compress(data)) +} + +// SaveFile stores the content of the file on the backend as a Blob by calling +// Save for each chunk. +func (ch *ContentHandler) SaveFile(filename string, size uint) (Blobs, error) { + file, err := os.Open(filename) + defer file.Close() + if err != nil { + return nil, err + } + + // if the file is small enough, store it directly + if size < chunker.MinSize { + buf, err := ioutil.ReadAll(file) + if err != nil { + return nil, err + } + + blob, err := ch.Save(backend.Blob, buf) + if err != nil { + return nil, err + } + + return Blobs{blob}, nil + } + + // else store all chunks + blobs := Blobs{} + chunker := chunker.New(file) + + for { + chunk, err := chunker.Next() + if err == io.EOF { + break + } + + if err != nil { + return nil, err + } + + blob, err := ch.Save(backend.Blob, chunk.Data) + if err != nil { + return nil, err + } + + blobs = append(blobs, blob) + } + + return blobs, nil +} + +// Load tries to load and decrypt content identified by t and id from the backend. +func (ch *ContentHandler) Load(t backend.Type, id backend.ID) ([]byte, error) { + if t == backend.Snapshot { + // load data + buf, err := ch.be.Get(t, id) + if err != nil { + return nil, err + } + + // decrypt + buf, err = ch.key.Decrypt(buf) + if err != nil { + return nil, err + } + + return buf, nil + } + + // lookup storage hash + blob := ch.content.Find(id) + if blob == nil { + return nil, errors.New("Storage ID not found") + } + + // load data + buf, err := ch.be.Get(t, blob.Storage) + if err != nil { + return nil, err + } + + // check length + if len(buf) != int(blob.StorageSize) { + return nil, errors.New("Invalid storage length") + } + + // decrypt + buf, err = ch.key.Decrypt(buf) + if err != nil { + return nil, err + } + + // check length + if len(buf) != int(blob.Size) { + return nil, errors.New("Invalid length") + } + + return buf, nil +} + +// LoadJSON calls Load() to get content from the backend and afterwards calls +// json.Unmarshal on the item. +func (ch *ContentHandler) LoadJSON(t backend.Type, id backend.ID, item interface{}) error { + // load from backend + buf, err := ch.Load(t, id) + if err != nil { + return err + } + + // inflate and unmarshal + err = json.Unmarshal(backend.Uncompress(buf), item) + return err +} + +// LoadJSONRaw loads data with the given storage id and type from the backend, +// decrypts it and calls json.Unmarshal on the item. +func (ch *ContentHandler) LoadJSONRaw(t backend.Type, id backend.ID, item interface{}) error { + // load data + buf, err := ch.be.Get(t, id) + if err != nil { + return err + } + + // decrypt + buf, err = ch.key.Decrypt(buf) + if err != nil { + return err + } + + // inflate and unmarshal + err = json.Unmarshal(backend.Uncompress(buf), item) + return err +} diff --git a/key.go b/key.go new file mode 100644 index 000000000..690934d37 --- /dev/null +++ b/key.go @@ -0,0 +1,388 @@ +package khepri + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/user" + "time" + + "github.com/fd0/khepri/backend" + + "code.google.com/p/go.crypto/scrypt" +) + +var ( + ErrUnauthenticated = errors.New("Ciphertext verification failed") + ErrNoKeyFound = errors.New("No key could be found") +) + +// TODO: figure out scrypt values on the fly depending on the current +// hardware. +const ( + scrypt_N = 65536 + scrypt_r = 8 + scrypt_p = 1 + scrypt_saltsize = 64 + aesKeysize = 32 // for AES256 + hmacKeysize = 32 // for HMAC with SHA256 +) + +type Key struct { + Created time.Time `json:"created"` + Username string `json:"username"` + Hostname string `json:"hostname"` + Comment string `json:"comment,omitempty"` + + KDF string `json:"kdf"` + N int `json:"N"` + R int `json:"r"` + P int `json:"p"` + Salt []byte `json:"salt"` + Data []byte `json:"data"` + + user *keys + master *keys +} + +type keys struct { + Sign []byte + Encrypt []byte +} + +func CreateKey(be backend.Server, password string) (*Key, error) { + // fill meta data about key + k := &Key{ + Created: time.Now(), + KDF: "scrypt", + N: scrypt_N, + R: scrypt_r, + P: scrypt_p, + } + + hn, err := os.Hostname() + if err == nil { + k.Hostname = hn + } + + usr, err := user.Current() + if err == nil { + k.Username = usr.Username + } + + // generate random salt + k.Salt = make([]byte, scrypt_saltsize) + n, err := rand.Read(k.Salt) + if n != scrypt_saltsize || err != nil { + panic("unable to read enough random bytes for salt") + } + + // call scrypt() to derive user key + k.user, err = k.scrypt(password) + if err != nil { + return nil, err + } + + // generate new random master keys + k.master, err = k.newKeys() + if err != nil { + return nil, err + } + + // encrypt master keys (as json) with user key + buf, err := json.Marshal(k.master) + if err != nil { + return nil, err + } + + k.Data, err = k.EncryptUser(buf) + + // dump as json + buf, err = json.Marshal(k) + if err != nil { + return nil, err + } + + // store in repository and return + _, err = be.Create(backend.Key, buf) + if err != nil { + return nil, err + } + + return k, nil +} + +func OpenKey(be backend.Server, id backend.ID, password string) (*Key, error) { + // extract data from repo + data, err := be.Get(backend.Key, id) + if err != nil { + return nil, err + } + + // restore json + k := &Key{} + err = json.Unmarshal(data, k) + if err != nil { + return nil, err + } + + // check KDF + if k.KDF != "scrypt" { + return nil, errors.New("only supported KDF is scrypt()") + } + + // derive user key + k.user, err = k.scrypt(password) + if err != nil { + return nil, err + } + + // decrypt master keys + buf, err := k.DecryptUser(k.Data) + if err != nil { + return nil, err + } + + // restore json + k.master = &keys{} + err = json.Unmarshal(buf, k.master) + if err != nil { + return nil, err + } + + return k, nil +} + +func SearchKey(be backend.Server, password string) (*Key, error) { + // list all keys + ids, err := be.List(backend.Key) + if err != nil { + panic(err) + } + + // try all keys in repo + var key *Key + for _, id := range ids { + key, err = OpenKey(be, id, password) + if err != nil { + continue + } + + return key, nil + } + + return nil, ErrNoKeyFound +} + +func (k *Key) scrypt(password string) (*keys, error) { + if len(k.Salt) == 0 { + return nil, fmt.Errorf("scrypt() called with empty salt") + } + + keybytes := hmacKeysize + aesKeysize + scrypt_keys, err := scrypt.Key([]byte(password), k.Salt, k.N, k.R, k.P, keybytes) + if err != nil { + return nil, fmt.Errorf("error deriving keys from password: %v", err) + } + + if len(scrypt_keys) != keybytes { + return nil, fmt.Errorf("invalid numbers of bytes expanded from scrypt(): %d", len(scrypt_keys)) + } + + ks := &keys{ + Encrypt: scrypt_keys[:aesKeysize], + Sign: scrypt_keys[aesKeysize:], + } + return ks, nil +} + +func (k *Key) newKeys() (*keys, error) { + ks := &keys{ + Encrypt: make([]byte, aesKeysize), + Sign: make([]byte, hmacKeysize), + } + n, err := rand.Read(ks.Encrypt) + if n != aesKeysize || err != nil { + panic("unable to read enough random bytes for encryption key") + } + n, err = rand.Read(ks.Sign) + if n != hmacKeysize || err != nil { + panic("unable to read enough random bytes for signing key") + } + + return ks, nil +} + +func (k *Key) newIV() ([]byte, error) { + buf := make([]byte, aes.BlockSize) + _, err := io.ReadFull(rand.Reader, buf) + if err != nil { + return nil, err + } + + return buf, nil +} + +func (k *Key) pad(plaintext []byte) []byte { + l := aes.BlockSize - (len(plaintext) % aes.BlockSize) + if l == 0 { + l = aes.BlockSize + } + + if l <= 0 || l > aes.BlockSize { + panic("invalid padding size") + } + + return append(plaintext, bytes.Repeat([]byte{byte(l)}, l)...) +} + +func (k *Key) unpad(plaintext []byte) []byte { + l := len(plaintext) + pad := plaintext[l-1] + + if pad > aes.BlockSize { + panic(errors.New("padding > BlockSize")) + } + + if pad == 0 { + panic(errors.New("invalid padding 0")) + } + + for i := l - int(pad); i < l; i++ { + if plaintext[i] != pad { + panic(errors.New("invalid padding!")) + } + } + + return plaintext[:l-int(pad)] +} + +// Encrypt encrypts and signs data. Returned is IV || Ciphertext || HMAC. For +// the hash function, SHA256 is used, so the overhead is 16+32=48 byte. +func (k *Key) encrypt(ks *keys, plaintext []byte) ([]byte, error) { + iv, err := k.newIV() + if err != nil { + panic(fmt.Sprintf("unable to generate new random iv: %v", err)) + } + + c, err := aes.NewCipher(ks.Encrypt) + if err != nil { + panic(fmt.Sprintf("unable to create cipher: %v", err)) + } + + e := cipher.NewCBCEncrypter(c, iv) + p := k.pad(plaintext) + ciphertext := make([]byte, len(p)) + e.CryptBlocks(ciphertext, p) + + ciphertext = append(iv, ciphertext...) + + hm := hmac.New(sha256.New, ks.Sign) + + n, err := hm.Write(ciphertext) + if err != nil || n != len(ciphertext) { + panic(fmt.Sprintf("unable to calculate hmac of ciphertext: %v", err)) + } + + return hm.Sum(ciphertext), nil +} + +// EncryptUser encrypts and signs data with the user key. Returned is IV || +// Ciphertext || HMAC. For the hash function, SHA256 is used, so the overhead +// is 16+32=48 byte. +func (k *Key) EncryptUser(plaintext []byte) ([]byte, error) { + return k.encrypt(k.user, plaintext) +} + +// Encrypt encrypts and signs data with the master key. Returned is IV || +// Ciphertext || HMAC. For the hash function, SHA256 is used, so the overhead +// is 16+32=48 byte. +func (k *Key) Encrypt(plaintext []byte) ([]byte, error) { + return k.encrypt(k.master, plaintext) +} + +// Decrypt verifes and decrypts the ciphertext. Ciphertext must be in the form +// IV || Ciphertext || HMAC. +func (k *Key) decrypt(ks *keys, ciphertext []byte) ([]byte, error) { + hm := hmac.New(sha256.New, ks.Sign) + + // extract hmac + l := len(ciphertext) - hm.Size() + ciphertext, mac := ciphertext[:l], ciphertext[l:] + + // calculate new hmac + n, err := hm.Write(ciphertext) + if err != nil || n != len(ciphertext) { + panic(fmt.Sprintf("unable to calculate hmac of ciphertext, err %v", err)) + } + + // verify hmac + mac2 := hm.Sum(nil) + + if !hmac.Equal(mac, mac2) { + return nil, ErrUnauthenticated + } + + // extract iv + iv, ciphertext := ciphertext[:aes.BlockSize], ciphertext[aes.BlockSize:] + + // decrypt data + c, err := aes.NewCipher(ks.Encrypt) + if err != nil { + panic(fmt.Sprintf("unable to create cipher: %v", err)) + } + + // decrypt + e := cipher.NewCBCDecrypter(c, iv) + plaintext := make([]byte, len(ciphertext)) + e.CryptBlocks(plaintext, ciphertext) + + // remove padding and return + return k.unpad(plaintext), nil +} + +// Decrypt verifes and decrypts the ciphertext with the master key. Ciphertext +// must be in the form IV || Ciphertext || HMAC. +func (k *Key) Decrypt(ciphertext []byte) ([]byte, error) { + return k.decrypt(k.master, ciphertext) +} + +// DecryptUser verifes and decrypts the ciphertext with the master key. Ciphertext +// must be in the form IV || Ciphertext || HMAC. +func (k *Key) DecryptUser(ciphertext []byte) ([]byte, error) { + return k.decrypt(k.user, ciphertext) +} + +// Each calls backend.Each() with the given parameters, Decrypt() on the +// ciphertext and, on successful decryption, f with the plaintext. +func (k *Key) Each(be backend.Server, t backend.Type, f func(backend.ID, []byte, error)) error { + return backend.Each(be, t, func(id backend.ID, data []byte, e error) { + if e != nil { + f(id, nil, e) + return + } + + buf, err := k.Decrypt(data) + if err != nil { + f(id, nil, err) + return + } + + f(id, buf, nil) + }) +} + +func (k *Key) String() string { + if k == nil { + return "" + } + return fmt.Sprintf("", k.Username, k.Hostname, k.Created) +} diff --git a/key_int_test.go b/key_int_test.go new file mode 100644 index 000000000..06032a83e --- /dev/null +++ b/key_int_test.go @@ -0,0 +1,79 @@ +package khepri + +import ( + "bytes" + "encoding/hex" + "testing" +) + +var test_values = []struct { + ekey, skey []byte + ciphertext []byte + plaintext []byte + should_panic bool +}{ + { + ekey: decode_hex("303e8687b1d7db18421bdc6bb8588ccadac4d59ee87b8ff70c44e635790cafef"), + skey: decode_hex("cc8d4b948ee0ebfe1d415de921d10353ef4d8824cb80b2bcc5fbff8a9b12a42c"), + ciphertext: decode_hex("154f582d77e6430409da392c3a09aa38e00a78bcc8919557fe18dd17f83e7b0b3053def59f4215b6e1c6b72ceb5acdddd8511ce3a853e054218de1e9f34637470d68f1f93ba8228e4d9817d7c9acfcd2"), + plaintext: []byte("Dies ist ein Test!"), + }, +} + +func decode_hex(s string) []byte { + d, _ := hex.DecodeString(s) + return d +} + +// returns true if function called panic +func should_panic(f func()) (did_panic bool) { + defer func() { + if r := recover(); r != nil { + did_panic = true + } + }() + + f() + + return false +} + +func TestCrypto(t *testing.T) { + r := &Key{} + + for _, tv := range test_values { + // test encryption + r.master = &keys{ + Encrypt: tv.ekey, + Sign: tv.skey, + } + + msg, err := r.encrypt(r.master, tv.plaintext) + if err != nil { + t.Fatal(err) + } + + // decrypt message + _, err = r.decrypt(r.master, msg) + if err != nil { + t.Fatal(err) + } + + // change hmac, this must fail + msg[len(msg)-8] ^= 0x23 + + if _, err = r.decrypt(r.master, msg); err != ErrUnauthenticated { + t.Fatal("wrong HMAC value not detected") + } + + // test decryption + p, err := r.decrypt(r.master, tv.ciphertext) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(p, tv.plaintext) { + t.Fatalf("wrong plaintext: expected %q but got %q\n", tv.plaintext, p) + } + } +} diff --git a/key_test.go b/key_test.go new file mode 100644 index 000000000..bf0e9db74 --- /dev/null +++ b/key_test.go @@ -0,0 +1,50 @@ +package khepri_test + +import ( + "flag" + "io/ioutil" + "os" + "testing" + + "github.com/fd0/khepri" + "github.com/fd0/khepri/backend" +) + +var test_password = "foobar" +var testCleanup = flag.Bool("test.cleanup", true, "clean up after running tests (remove local backend directory with all content)") + +func setupBackend(t *testing.T) *backend.Local { + tempdir, err := ioutil.TempDir("", "khepri-test-") + ok(t, err) + + b, err := backend.CreateLocal(tempdir) + ok(t, err) + + t.Logf("created local backend at %s", tempdir) + + return b +} + +func teardownBackend(t *testing.T, b *backend.Local) { + if !*testCleanup { + t.Logf("leaving local backend at %s\n", b.Location()) + return + } + + ok(t, os.RemoveAll(b.Location())) +} + +func setupKey(t *testing.T, be backend.Server, password string) *khepri.Key { + c, err := khepri.CreateKey(be, password) + ok(t, err) + + t.Logf("created Safe at %s", be.Location()) + + return c +} + +func TestSafe(t *testing.T) { + be := setupBackend(t) + defer teardownBackend(t, be) + _ = setupKey(t, be, test_password) +} diff --git a/object.go b/object.go deleted file mode 100644 index 8e404eba5..000000000 --- a/object.go +++ /dev/null @@ -1,27 +0,0 @@ -package khepri - -func (repo *Repository) Create(t Type, data []byte) (ID, error) { - // TODO: make sure that tempfile is removed upon error - - // create tempfile in repository - var err error - file, err := repo.tempFile() - if err != nil { - return nil, err - } - - // write data to tempfile - _, err = file.Write(data) - if err != nil { - return nil, err - } - - // close tempfile, return id - id := IDFromData(data) - err = repo.renameFile(file, t, id) - if err != nil { - return nil, err - } - - return id, nil -} diff --git a/object_test.go b/object_test.go deleted file mode 100644 index 1ad672273..000000000 --- a/object_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package khepri_test - -import ( - "testing" - - "github.com/fd0/khepri" -) - -func TestObjects(t *testing.T) { - repo, err := setupRepo() - ok(t, err) - - defer func() { - err = teardownRepo(repo) - ok(t, err) - }() - - for _, test := range TestStrings { - id, err := repo.Create(khepri.TYPE_BLOB, []byte(test.data)) - ok(t, err) - - id2, err := khepri.ParseID(test.id) - ok(t, err) - - equals(t, id2, id) - } -} diff --git a/repository.go b/repository.go deleted file mode 100644 index 61c977b69..000000000 --- a/repository.go +++ /dev/null @@ -1,326 +0,0 @@ -package khepri - -import ( - "crypto/rand" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "hash" - "io" - "io/ioutil" - "net/url" - "os" - "path" - "path/filepath" -) - -const ( - dirMode = 0700 - blobPath = "blobs" - refPath = "refs" - tempPath = "tmp" - configFileName = "config.json" -) - -var ( - ErrIDDoesNotExist = errors.New("ID does not exist") -) - -// Name stands for the alias given to an ID. -type Name string - -func (n Name) Encode() string { - return url.QueryEscape(string(n)) -} - -type HashFunc func() hash.Hash - -type Repository struct { - path string - hash HashFunc - config *Config -} - -type Config struct { - Salt string - N uint - R uint `json:"r"` - P uint `json:"p"` -} - -// TODO: figure out scrypt values on the fly depending on the current -// hardware. -const ( - scrypt_N = 65536 - scrypt_r = 8 - scrypt_p = 1 - scrypt_saltsize = 64 -) - -type Type int - -const ( - TYPE_BLOB = iota - TYPE_REF -) - -func NewTypeFromString(s string) Type { - switch s { - case "blob": - return TYPE_BLOB - case "ref": - return TYPE_REF - } - - panic(fmt.Sprintf("unknown type %q", s)) -} - -func (t Type) String() string { - switch t { - case TYPE_BLOB: - return "blob" - case TYPE_REF: - return "ref" - } - - panic(fmt.Sprintf("unknown type %d", t)) -} - -// NewRepository opens a dir-baked repository at the given path. -func NewRepository(path string) (*Repository, error) { - var err error - - d := &Repository{ - path: path, - hash: sha256.New, - } - - d.config, err = d.read_config() - if err != nil { - return nil, err - } - - return d, nil -} - -func (r *Repository) read_config() (*Config, error) { - // try to open config file - f, err := os.Open(path.Join(r.path, configFileName)) - if err != nil { - return nil, err - } - - cfg := new(Config) - buf, err := ioutil.ReadAll(f) - if err != nil { - return nil, err - } - - err = json.Unmarshal(buf, cfg) - if err != nil { - return nil, err - } - - return cfg, nil -} - -// CreateRepository creates all the necessary files and directories for the -// Repository. -func CreateRepository(p string) (*Repository, error) { - dirs := []string{ - p, - path.Join(p, blobPath), - path.Join(p, refPath), - path.Join(p, tempPath), - } - - var configfile = path.Join(p, configFileName) - - // test if repository directories or config file already exist - if _, err := os.Stat(configfile); err == nil { - return nil, fmt.Errorf("config file %s already exists", configfile) - } - - for _, d := range dirs[1:] { - if _, err := os.Stat(d); err == nil { - return nil, fmt.Errorf("dir %s already exists", d) - } - } - - // create initial json configuration - cfg := &Config{ - N: scrypt_N, - R: scrypt_r, - P: scrypt_p, - } - - // generate salt - buf := make([]byte, scrypt_saltsize) - n, err := rand.Read(buf) - if n != scrypt_saltsize || err != nil { - panic("unable to read enough random bytes for salt") - } - cfg.Salt = hex.EncodeToString(buf) - - // create ps for blobs, refs and temp - for _, dir := range dirs { - err := os.MkdirAll(dir, dirMode) - if err != nil { - return nil, err - } - } - - // write config file - f, err := os.Create(configfile) - defer f.Close() - if err != nil { - return nil, err - } - - s, err := json.Marshal(cfg) - if err != nil { - return nil, err - } - - _, err = f.Write(s) - if err != nil { - return nil, err - } - - // open repository - return NewRepository(p) -} - -// SetHash changes the hash function used for deriving IDs. Default is SHA256. -func (r *Repository) SetHash(h HashFunc) { - r.hash = h -} - -// Path returns the directory used for this repository. -func (r *Repository) Path() string { - return r.path -} - -// Return temp directory in correct directory for this repository. -func (r *Repository) tempFile() (*os.File, error) { - return ioutil.TempFile(path.Join(r.path, tempPath), "temp-") -} - -// Rename temp file to final name according to type and ID. -func (r *Repository) renameFile(file *os.File, t Type, id ID) error { - filename := path.Join(r.dir(t), id.String()) - return os.Rename(file.Name(), filename) -} - -// Construct directory for given Type. -func (r *Repository) dir(t Type) string { - switch t { - case TYPE_BLOB: - return path.Join(r.path, blobPath) - case TYPE_REF: - return path.Join(r.path, refPath) - } - - panic(fmt.Sprintf("unknown type %d", t)) -} - -// Construct path for given Type and ID. -func (r *Repository) filename(t Type, id ID) string { - return path.Join(r.dir(t), id.String()) -} - -// Test returns true if the given ID exists in the repository. -func (r *Repository) Test(t Type, id ID) (bool, error) { - // try to open file - file, err := os.Open(r.filename(t, id)) - defer func() { - file.Close() - }() - - if err != nil { - if os.IsNotExist(err) { - return false, nil - } - return false, err - } - - return true, nil -} - -// Get returns a reader for the content stored under the given ID. -func (r *Repository) Get(t Type, id ID) (io.ReadCloser, error) { - // try to open file - file, err := os.Open(r.filename(t, id)) - if err != nil { - return nil, err - } - - return file, nil -} - -// Remove removes the content stored at ID. -func (r *Repository) Remove(t Type, id ID) error { - return os.Remove(r.filename(t, id)) -} - -type IDs []ID - -// Lists all objects of a given type. -func (r *Repository) List(t Type) (IDs, error) { - // TODO: use os.Open() and d.Readdirnames() instead of Glob() - pattern := path.Join(r.dir(t), "*") - - matches, err := filepath.Glob(pattern) - if err != nil { - return nil, err - } - - ids := make(IDs, 0, len(matches)) - - for _, m := range matches { - base := filepath.Base(m) - - if base == "" { - continue - } - id, err := ParseID(base) - - if err != nil { - continue - } - - ids = append(ids, id) - } - - return ids, nil -} - -func (ids IDs) Len() int { - return len(ids) -} - -func (ids IDs) Less(i, j int) bool { - if len(ids[i]) < len(ids[j]) { - return true - } - - for k, b := range ids[i] { - if b == ids[j][k] { - continue - } - - if b < ids[j][k] { - return true - } else { - return false - } - } - - return false -} - -func (ids IDs) Swap(i, j int) { - ids[i], ids[j] = ids[j], ids[i] -} diff --git a/repository_test.go b/repository_test.go deleted file mode 100644 index ee8337520..000000000 --- a/repository_test.go +++ /dev/null @@ -1,130 +0,0 @@ -package khepri_test - -import ( - "flag" - "fmt" - "io/ioutil" - "os" - "sort" - "testing" - - "github.com/fd0/khepri" -) - -var testCleanup = flag.Bool("test.cleanup", true, "clean up after running tests (remove repository directory with all content)") - -var TestStrings = []struct { - id string - t khepri.Type - data string -}{ - {"c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2", khepri.TYPE_BLOB, "foobar"}, - {"248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", khepri.TYPE_BLOB, "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"}, - {"cc5d46bdb4991c6eae3eb739c9c8a7a46fe9654fab79c47b4fe48383b5b25e1c", khepri.TYPE_REF, "foo/bar"}, - {"4e54d2c721cbdb730f01b10b62dec622962b36966ec685880effa63d71c808f2", khepri.TYPE_BLOB, "foo/../../baz"}, -} - -func setupRepo() (*khepri.Repository, error) { - tempdir, err := ioutil.TempDir("", "khepri-test-") - if err != nil { - return nil, err - } - - repo, err := khepri.CreateRepository(tempdir) - if err != nil { - return nil, err - } - - return repo, nil -} - -func teardownRepo(repo *khepri.Repository) error { - if !*testCleanup { - fmt.Fprintf(os.Stderr, "leaving repository at %s\n", repo.Path()) - return nil - } - - err := os.RemoveAll(repo.Path()) - if err != nil { - return err - } - - return nil -} - -func TestRepository(t *testing.T) { - repo, err := setupRepo() - ok(t, err) - - defer func() { - err = teardownRepo(repo) - ok(t, err) - }() - - // detect non-existing files - for _, test := range TestStrings { - id, err := khepri.ParseID(test.id) - ok(t, err) - - // try to get string out, should fail - ret, err := repo.Test(test.t, id) - ok(t, err) - assert(t, !ret, fmt.Sprintf("id %q was found (but should not have)", test.id)) - } - - // add files - for _, test := range TestStrings { - // store string in repository - id, err := repo.Create(test.t, []byte(test.data)) - ok(t, err) - - equals(t, test.id, id.String()) - - // try to get it out again - rd, err := repo.Get(test.t, id) - ok(t, err) - assert(t, rd != nil, "Get() returned nil reader") - - // compare content - buf, err := ioutil.ReadAll(rd) - equals(t, test.data, string(buf)) - } - - // list ids - for _, tpe := range []khepri.Type{khepri.TYPE_BLOB, khepri.TYPE_REF} { - IDs := khepri.IDs{} - for _, test := range TestStrings { - if test.t == tpe { - id, err := khepri.ParseID(test.id) - ok(t, err) - IDs = append(IDs, id) - } - } - - ids, err := repo.List(tpe) - ok(t, err) - - sort.Sort(ids) - sort.Sort(IDs) - equals(t, IDs, ids) - } - - // remove content if requested - if *testCleanup { - for _, test := range TestStrings { - id, err := khepri.ParseID(test.id) - ok(t, err) - - found, err := repo.Test(test.t, id) - ok(t, err) - assert(t, found, fmt.Sprintf("id %q was not found before removal")) - - err = repo.Remove(test.t, id) - ok(t, err) - - found, err = repo.Test(test.t, id) - ok(t, err) - assert(t, !found, fmt.Sprintf("id %q was not found before removal")) - } - } -} diff --git a/restorer.go b/restorer.go new file mode 100644 index 000000000..1b3f3ab58 --- /dev/null +++ b/restorer.go @@ -0,0 +1,100 @@ +package khepri + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/fd0/khepri/backend" +) + +type Restorer struct { + be backend.Server + key *Key + ch *ContentHandler + sn *Snapshot + + Error func(dir string, node *Node, err error) error + Filter func(item string, node *Node) bool +} + +// NewRestorer creates a restorer preloaded with the content from the snapshot snid. +func NewRestorer(be backend.Server, key *Key, snid backend.ID) (*Restorer, error) { + r := &Restorer{ + be: be, + key: key, + } + + var err error + r.ch, err = NewContentHandler(be, key) + if err != nil { + return nil, err + } + + r.sn, err = r.ch.LoadSnapshot(snid) + if err != nil { + return nil, err + } + + // abort on all errors + r.Error = func(string, *Node, error) error { return err } + // allow all files + r.Filter = func(string, *Node) bool { return true } + + return r, nil +} + +func (res *Restorer) to(dir string, tree_id backend.ID) error { + tree := Tree{} + err := res.ch.LoadJSON(backend.Tree, tree_id, &tree) + if err != nil { + return res.Error(dir, nil, err) + } + + for _, node := range tree { + p := filepath.Join(dir, node.Name) + if !res.Filter(p, node) { + continue + } + + err := node.CreateAt(res.ch, p) + if err != nil { + err = res.Error(p, node, err) + if err != nil { + return err + } + } + + if node.Type == "dir" { + if node.Subtree == nil { + return errors.New(fmt.Sprintf("Dir without subtree in tree %s", tree_id)) + } + + err = res.to(p, node.Subtree) + if err != nil { + err = res.Error(p, node, err) + if err != nil { + return err + } + } + } + } + + return nil +} + +// RestoreTo creates the directories and files in the snapshot below dir. +// Before an item is created, res.Filter is called. +func (res *Restorer) RestoreTo(dir string) error { + err := os.MkdirAll(dir, 0700) + if err != nil && err != os.ErrExist { + return err + } + + return res.to(dir, res.sn.Content) +} + +func (res *Restorer) Snapshot() *Snapshot { + return res.sn +} diff --git a/snapshot.go b/snapshot.go index 21ccde0a2..fa7d10e1d 100644 --- a/snapshot.go +++ b/snapshot.go @@ -1,29 +1,36 @@ package khepri import ( - "encoding/json" "fmt" "os" "os/user" + "path/filepath" "time" + + "github.com/fd0/khepri/backend" ) type Snapshot struct { - Time time.Time `json:"time"` - Content ID `json:"content"` - Tree *Tree `json:"-"` - Dir string `json:"dir"` - Hostname string `json:"hostname,omitempty"` - Username string `json:"username,omitempty"` - UID string `json:"uid,omitempty"` - GID string `json:"gid,omitempty"` - id ID `json:omit` - repo *Repository + Time time.Time `json:"time"` + Content backend.ID `json:"content"` + StorageMap *StorageMap `json:"map"` + Dir string `json:"dir"` + Hostname string `json:"hostname,omitempty"` + Username string `json:"username,omitempty"` + UID string `json:"uid,omitempty"` + GID string `json:"gid,omitempty"` + + id backend.ID // plaintext ID, used during restore } func NewSnapshot(dir string) *Snapshot { + d, err := filepath.Abs(dir) + if err != nil { + d = dir + } + sn := &Snapshot{ - Dir: dir, + Dir: d, Time: time.Now(), } @@ -42,66 +49,16 @@ func NewSnapshot(dir string) *Snapshot { return sn } -func (sn *Snapshot) Save(repo *Repository) (ID, error) { - if sn.Content == nil { - panic("Snapshot.Save() called with nil tree id") - } - - data, err := json.Marshal(sn) - if err != nil { - return nil, err - } - - id, err := repo.Create(TYPE_REF, data) - if err != nil { - return nil, err - } - - return id, nil -} - -func LoadSnapshot(repo *Repository, id ID) (*Snapshot, error) { - rd, err := repo.Get(TYPE_REF, id) - if err != nil { - return nil, err - } - - // TODO: maybe inject a hashing reader here and test if the given id is correct - - dec := json.NewDecoder(rd) +func LoadSnapshot(ch *ContentHandler, id backend.ID) (*Snapshot, error) { sn := &Snapshot{} - err = dec.Decode(sn) - + err := ch.LoadJSON(backend.Snapshot, id, sn) if err != nil { return nil, err } - sn.id = id - sn.repo = repo - return sn, nil } -func (sn *Snapshot) RestoreAt(path string) error { - err := os.MkdirAll(path, 0700) - if err != nil { - return err - } - - if sn.Tree == nil { - sn.Tree, err = NewTreeFromRepo(sn.repo, sn.Content) - if err != nil { - return err - } - } - - return sn.Tree.CreateAt(path) -} - -func (sn *Snapshot) ID() ID { - return sn.id -} - func (sn *Snapshot) String() string { - return fmt.Sprintf("", sn.Dir, sn.Time.Format(time.RFC822Z)) + return fmt.Sprintf("", sn.Dir, sn.Time) } diff --git a/snapshot_test.go b/snapshot_test.go index 622407377..d06ec077c 100644 --- a/snapshot_test.go +++ b/snapshot_test.go @@ -5,23 +5,24 @@ import ( "time" "github.com/fd0/khepri" + "github.com/fd0/khepri/backend" ) -func TestSnapshot(t *testing.T) { - repo, err := setupRepo() - ok(t, err) - - defer func() { - err = teardownRepo(repo) - ok(t, err) - }() - +func testSnapshot(t *testing.T, be backend.Server) { + var err error sn := khepri.NewSnapshot("/home/foobar") - sn.Content, err = khepri.ParseID("c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2") + sn.Content, err = backend.ParseID("c3ab8ff13720e8ad9047dd39466b3c8974e592c2fa383d4a3960714caef0c4f2") ok(t, err) sn.Time, err = time.Parse(time.RFC3339Nano, "2014-08-03T17:49:05.378595539+02:00") ok(t, err) - _, err = sn.Save(repo) - ok(t, err) + // _, err = sn.Save(be) + // ok(t, err) +} + +func TestSnapshot(t *testing.T) { + repo := setupBackend(t) + defer teardownBackend(t, repo) + + testSnapshot(t, repo) } diff --git a/storagemap.go b/storagemap.go new file mode 100644 index 000000000..db77952e2 --- /dev/null +++ b/storagemap.go @@ -0,0 +1,51 @@ +package khepri + +import ( + "bytes" + "sort" + + "github.com/fd0/khepri/backend" +) + +type StorageMap Blobs + +func NewStorageMap() *StorageMap { + return &StorageMap{} +} + +func (m StorageMap) find(id backend.ID) (int, *Blob) { + i := sort.Search(len(m), func(i int) bool { + return bytes.Compare(m[i].ID, id) >= 0 + }) + + if i < len(m) && bytes.Equal(m[i].ID, id) { + return i, m[i] + } + + return i, nil +} + +func (m StorageMap) Find(id backend.ID) *Blob { + _, blob := m.find(id) + return blob +} + +func (m *StorageMap) Insert(blob *Blob) { + pos, b := m.find(blob.ID) + if b != nil { + // already present + return + } + + // insert blob + // https://code.google.com/p/go-wiki/wiki/SliceTricks + *m = append(*m, nil) + copy((*m)[pos+1:], (*m)[pos:]) + (*m)[pos] = blob +} + +func (m *StorageMap) Merge(sm *StorageMap) { + for _, blob := range *sm { + m.Insert(blob) + } +} diff --git a/test/run.sh b/test/run.sh index e4d197b4d..7455e5079 100755 --- a/test/run.sh +++ b/test/run.sh @@ -11,6 +11,7 @@ prepare() { export BASE="$(mktemp --tmpdir --directory khepri-testsuite-XXXXXX)" export KHEPRI_REPOSITORY="${BASE}/khepri-backup" export DATADIR="${BASE}/fake-data" + export KHEPRI_PASSWORD="foobar" debug "repository is at ${KHEPRI_REPOSITORY}" mkdir -p "$DATADIR" diff --git a/test/test-backup.sh b/test/test-backup.sh index c41f3670e..f5d938bab 100755 --- a/test/test-backup.sh +++ b/test/test-backup.sh @@ -3,6 +3,6 @@ set -e prepare run khepri init run khepri backup "${BASE}/fake-data" -run khepri restore "$(khepri list ref)" "${BASE}/fake-data-restore" -dirdiff "${BASE}/fake-data" "${BASE}/fake-data-restore" +run khepri restore "$(khepri snapshots)" "${BASE}/fake-data-restore" +dirdiff "${BASE}/fake-data" "${BASE}/fake-data-restore/fake-data" cleanup diff --git a/tree.go b/tree.go index 29464c711..17c91a486 100644 --- a/tree.go +++ b/tree.go @@ -1,248 +1,71 @@ package khepri import ( - "bytes" - "encoding/json" "fmt" - "io" - "io/ioutil" "os" "os/user" - "path/filepath" "strconv" + "strings" "syscall" "time" - "github.com/fd0/khepri/chunker" + "github.com/fd0/khepri/backend" ) -type Tree struct { - Nodes []*Node `json:"nodes,omitempty"` -} +type Tree []*Node type Node struct { - Name string `json:"name"` - Type string `json:"type"` - Mode os.FileMode `json:"mode,omitempty"` - ModTime time.Time `json:"mtime,omitempty"` - AccessTime time.Time `json:"atime,omitempty"` - ChangeTime time.Time `json:"ctime,omitempty"` - UID uint32 `json:"uid"` - GID uint32 `json:"gid"` - User string `json:"user,omitempty"` - Group string `json:"group,omitempty"` - Inode uint64 `json:"inode,omitempty"` - Size uint64 `json:"size,omitempty"` - Links uint64 `json:"links,omitempty"` - LinkTarget string `json:"linktarget,omitempty"` - Device uint64 `json:"device,omitempty"` - Content []ID `json:"content,omitempty"` - Subtree ID `json:"subtree,omitempty"` - Tree *Tree `json:"-"` - repo *Repository + Name string `json:"name"` + Type string `json:"type"` + Mode os.FileMode `json:"mode,omitempty"` + ModTime time.Time `json:"mtime,omitempty"` + AccessTime time.Time `json:"atime,omitempty"` + ChangeTime time.Time `json:"ctime,omitempty"` + UID uint32 `json:"uid"` + GID uint32 `json:"gid"` + User string `json:"user,omitempty"` + Group string `json:"group,omitempty"` + Inode uint64 `json:"inode,omitempty"` + Size uint64 `json:"size,omitempty"` + Links uint64 `json:"links,omitempty"` + LinkTarget string `json:"linktarget,omitempty"` + Device uint64 `json:"device,omitempty"` + Content []backend.ID `json:"content,omitempty"` + Subtree backend.ID `json:"subtree,omitempty"` + + path string } -func NewTree() *Tree { - return &Tree{ - Nodes: []*Node{}, - } +type Blob struct { + ID backend.ID `json:"id,omitempty"` + Size uint64 `json:"size,omitempty"` + Storage backend.ID `json:"sid,omitempty"` // encrypted ID + StorageSize uint64 `json:"ssize,omitempty"` // encrypted Size } -func store_chunk(repo *Repository, rd io.Reader) (ID, error) { - data, err := ioutil.ReadAll(rd) - if err != nil { - return nil, err +type Blobs []*Blob + +func (n Node) String() string { + switch n.Type { + case "file": + return fmt.Sprintf("%s %5d %5d %6d %s %s", + n.Mode, n.UID, n.GID, n.Size, n.ModTime, n.Name) + case "dir": + return fmt.Sprintf("%s %5d %5d %6d %s %s", + n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime, n.Name) } - id, err := repo.Create(TYPE_BLOB, data) - if err != nil { - return nil, err - } - - return id, nil + return fmt.Sprintf("", n.Type, n.Name) } -func NewTreeFromPath(repo *Repository, dir string) (*Tree, error) { - fd, err := os.Open(dir) - defer fd.Close() - if err != nil { - return nil, err +func (t Tree) String() string { + s := []string{} + for _, n := range t { + s = append(s, n.String()) } - - entries, err := fd.Readdir(-1) - if err != nil { - return nil, err - } - - tree := &Tree{ - Nodes: make([]*Node, 0, len(entries)), - } - - for _, entry := range entries { - path := filepath.Join(dir, entry.Name()) - node, err := NodeFromFileInfo(path, entry) - if err != nil { - return nil, err - } - node.repo = repo - - tree.Nodes = append(tree.Nodes, node) - - if entry.IsDir() { - node.Tree, err = NewTreeFromPath(repo, path) - if err != nil { - return nil, err - } - continue - } - - if node.Type == "file" { - file, err := os.Open(path) - defer file.Close() - if err != nil { - return nil, err - } - - if node.Size < chunker.MinSize { - // if the file is small enough, store it directly - id, err := store_chunk(repo, file) - - if err != nil { - return nil, err - } - - node.Content = []ID{id} - - } else { - // else store chunks - node.Content = []ID{} - ch := chunker.New(file) - - for { - chunk, err := ch.Next() - - if err == io.EOF { - break - } - - if err != nil { - return nil, err - } - - id, err := store_chunk(repo, bytes.NewBuffer(chunk.Data)) - - node.Content = append(node.Content, id) - } - - } - } - } - - return tree, nil + return strings.Join(s, "\n") } -func (tree *Tree) Save(repo *Repository) (ID, error) { - for _, node := range tree.Nodes { - if node.Tree != nil { - var err error - node.Subtree, err = node.Tree.Save(repo) - if err != nil { - return nil, err - } - } - } - - data, err := json.Marshal(tree) - if err != nil { - return nil, err - } - - id, err := repo.Create(TYPE_BLOB, data) - if err != nil { - return nil, err - } - - return id, nil -} - -func NewTreeFromRepo(repo *Repository, id ID) (*Tree, error) { - tree := NewTree() - - rd, err := repo.Get(TYPE_BLOB, id) - defer rd.Close() - if err != nil { - return nil, err - } - - decoder := json.NewDecoder(rd) - - err = decoder.Decode(tree) - if err != nil { - return nil, err - } - - for _, node := range tree.Nodes { - node.repo = repo - - if node.Subtree != nil { - node.Tree, err = NewTreeFromRepo(repo, node.Subtree) - if err != nil { - return nil, err - } - } - } - - return tree, nil -} - -func (tree *Tree) CreateAt(path string) error { - for _, node := range tree.Nodes { - nodepath := filepath.Join(path, node.Name) - - if node.Type == "dir" { - err := os.Mkdir(nodepath, 0700) - if err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err) - continue - } - - err = os.Chmod(nodepath, node.Mode) - if err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err) - continue - } - - err = os.Chown(nodepath, int(node.UID), int(node.GID)) - if err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err) - continue - } - - err = node.Tree.CreateAt(filepath.Join(path, node.Name)) - if err != nil { - return err - } - - err = os.Chtimes(nodepath, node.AccessTime, node.ModTime) - if err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err) - continue - } - - } else { - err := node.CreateAt(nodepath) - if err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err) - continue - } - } - } - - return nil -} - -// TODO: make sure that node.Type is valid - func (node *Node) fill_extra(path string, fi os.FileInfo) (err error) { stat, ok := fi.Sys().(*syscall.Stat_t) if !ok { @@ -290,6 +113,7 @@ func (node *Node) fill_extra(path string, fi os.FileInfo) (err error) { func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) { node := &Node{ + path: path, Name: fi.Name(), Mode: fi.Mode() & os.ModePerm, ModTime: fi.ModTime(), @@ -316,12 +140,27 @@ func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) { return node, err } -func (node *Node) CreateAt(path string) error { - if node.repo == nil { - return fmt.Errorf("repository is nil!") - } - +func (node *Node) CreateAt(ch *ContentHandler, path string) error { switch node.Type { + case "dir": + err := os.Mkdir(path, node.Mode) + if err != nil { + return err + } + + err = os.Lchown(path, int(node.UID), int(node.GID)) + if err != nil { + return err + } + + var utimes = []syscall.Timespec{ + syscall.NsecToTimespec(node.AccessTime.UnixNano()), + syscall.NsecToTimespec(node.ModTime.UnixNano()), + } + err = syscall.UtimesNano(path, utimes) + if err != nil { + return err + } case "file": // TODO: handle hard links f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600) @@ -331,18 +170,32 @@ func (node *Node) CreateAt(path string) error { } for _, blobid := range node.Content { - rd, err := node.repo.Get(TYPE_BLOB, blobid) + buf, err := ch.Load(backend.Blob, blobid) if err != nil { return err } - _, err = io.Copy(f, rd) + _, err = f.Write(buf) if err != nil { return err } } f.Close() + + err = os.Lchown(path, int(node.UID), int(node.GID)) + if err != nil { + return err + } + + var utimes = []syscall.Timespec{ + syscall.NsecToTimespec(node.AccessTime.UnixNano()), + syscall.NsecToTimespec(node.ModTime.UnixNano()), + } + err = syscall.UtimesNano(path, utimes) + if err != nil { + return err + } case "symlink": err := os.Symlink(node.LinkTarget, path) if err != nil { diff --git a/tree_test.go b/tree_test.go new file mode 100644 index 000000000..6cfb0ad69 --- /dev/null +++ b/tree_test.go @@ -0,0 +1,54 @@ +package khepri_test + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" +) + +var testFiles = []struct { + name string + content []byte +}{ + {"foo", []byte("bar")}, + {"bar/foo2", []byte("bar2")}, + {"bar/bla/blubb", []byte("This is just a test!\n")}, +} + +// prepare directory and return temporary path +func prepare_dir(t *testing.T) string { + tempdir, err := ioutil.TempDir("", "khepri-test-") + ok(t, err) + + for _, test := range testFiles { + file := filepath.Join(tempdir, test.name) + dir := filepath.Dir(file) + if dir != "." { + ok(t, os.MkdirAll(dir, 0755)) + } + + f, err := os.Create(file) + defer func() { + ok(t, f.Close()) + }() + + ok(t, err) + + _, err = f.Write(test.content) + ok(t, err) + } + + t.Logf("tempdir prepared at %s", tempdir) + + return tempdir +} + +func TestTree(t *testing.T) { + dir := prepare_dir(t) + defer func() { + if *testCleanup { + ok(t, os.RemoveAll(dir)) + } + }() +}