From 89ab6d557e01985035d299d411b571614591a68b Mon Sep 17 00:00:00 2001 From: DRON-666 <64691982+DRON-666@users.noreply.github.com> Date: Tue, 10 Nov 2020 03:44:03 +0300 Subject: [PATCH] dump: Add zip dumper --- internal/dump/zip.go | 64 +++++++++++++ internal/dump/zip_test.go | 195 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 259 insertions(+) create mode 100644 internal/dump/zip.go create mode 100644 internal/dump/zip_test.go diff --git a/internal/dump/zip.go b/internal/dump/zip.go new file mode 100644 index 000000000..1c1035b08 --- /dev/null +++ b/internal/dump/zip.go @@ -0,0 +1,64 @@ +package dump + +import ( + "archive/zip" + "context" + "io" + "path/filepath" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/restic" +) + +type zipDumper struct { + w *zip.Writer +} + +// Statically ensure that zipDumper implements dumper. +var _ dumper = tarDumper{} + +// WriteZip will write the contents of the given tree, encoded as a zip to the given destination. +func WriteZip(ctx context.Context, repo restic.Repository, tree *restic.Tree, rootPath string, dst io.Writer) error { + dmp := zipDumper{w: zip.NewWriter(dst)} + + err := writeDump(ctx, repo, tree, rootPath, dmp, dst) + if err != nil { + dmp.w.Close() + return err + } + + return dmp.w.Close() +} + +func (dmp zipDumper) dumpNode(ctx context.Context, node *restic.Node, repo restic.Repository) error { + relPath, err := filepath.Rel("/", node.Path) + if err != nil { + return err + } + + header := &zip.FileHeader{ + Name: filepath.ToSlash(relPath), + UncompressedSize64: node.Size, + Modified: node.ModTime, + } + header.SetMode(node.Mode) + + if IsDir(node) { + header.Name += "/" + } + + w, err := dmp.w.CreateHeader(header) + if err != nil { + return errors.Wrap(err, "ZipHeader ") + } + + if IsLink(node) { + if _, err = w.Write([]byte(node.LinkTarget)); err != nil { + return errors.Wrap(err, "Write") + } + + return nil + } + + return GetNodeData(ctx, w, repo, node) +} diff --git a/internal/dump/zip_test.go b/internal/dump/zip_test.go new file mode 100644 index 000000000..70ad00275 --- /dev/null +++ b/internal/dump/zip_test.go @@ -0,0 +1,195 @@ +package dump + +import ( + "archive/zip" + "bytes" + "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/restic/restic/internal/archiver" + "github.com/restic/restic/internal/fs" + rtest "github.com/restic/restic/internal/test" +) + +func TestWriteZip(t *testing.T) { + tests := []struct { + name string + args archiver.TestDir + target string + }{ + { + name: "single file in root", + args: archiver.TestDir{ + "file": archiver.TestFile{Content: "string"}, + }, + target: "/", + }, + { + name: "multiple files in root", + args: archiver.TestDir{ + "file1": archiver.TestFile{Content: "string"}, + "file2": archiver.TestFile{Content: "string"}, + }, + target: "/", + }, + { + name: "multiple files and folders in root", + args: archiver.TestDir{ + "file1": archiver.TestFile{Content: "string"}, + "file2": archiver.TestFile{Content: "string"}, + "firstDir": archiver.TestDir{ + "another": archiver.TestFile{Content: "string"}, + }, + "secondDir": archiver.TestDir{ + "another2": archiver.TestFile{Content: "string"}, + }, + }, + target: "/", + }, + { + name: "file and symlink in root", + args: archiver.TestDir{ + "file1": archiver.TestFile{Content: "string"}, + "file2": archiver.TestSymlink{Target: "file1"}, + }, + target: "/", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + tmpdir, repo, cleanup := prepareTempdirRepoSrc(t, tt.args) + defer cleanup() + + arch := archiver.New(repo, fs.Track{FS: fs.Local{}}, archiver.Options{}) + + back := rtest.Chdir(t, tmpdir) + defer back() + + sn, _, err := arch.Snapshot(ctx, []string{"."}, archiver.SnapshotOptions{}) + rtest.OK(t, err) + + tree, err := repo.LoadTree(ctx, *sn.Tree) + rtest.OK(t, err) + + dst := &bytes.Buffer{} + if err := WriteZip(ctx, repo, tree, tt.target, dst); err != nil { + t.Fatalf("WriteZip() error = %v", err) + } + if err := checkZip(t, tmpdir, dst); err != nil { + t.Errorf("WriteZip() = zip does not match: %v", err) + } + }) + } +} + +func readZipFile(f *zip.File) ([]byte, error) { + rc, err := f.Open() + if err != nil { + return nil, err + } + defer rc.Close() + + b := &bytes.Buffer{} + _, err = b.ReadFrom(rc) + if err != nil { + return nil, err + } + + return b.Bytes(), nil +} + +func checkZip(t *testing.T, testDir string, srcZip *bytes.Buffer) error { + z, err := zip.NewReader(bytes.NewReader(srcZip.Bytes()), int64(srcZip.Len())) + if err != nil { + return err + } + + fileNumber := 0 + zipFiles := len(z.File) + + err = filepath.Walk(testDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.Name() != filepath.Base(testDir) { + fileNumber++ + } + return nil + }) + if err != nil { + return err + } + + for _, f := range z.File { + matchPath := filepath.Join(testDir, f.Name) + match, err := os.Lstat(matchPath) + if err != nil { + return err + } + + // check metadata, zip header contains time rounded to seconds + fileTime := match.ModTime().Truncate(time.Second) + zipTime := f.Modified + if !fileTime.Equal(zipTime) { + return fmt.Errorf("modTime does not match, got: %s, want: %s", zipTime, fileTime) + } + if f.Mode() != match.Mode() { + return fmt.Errorf("mode does not match, got: %v [%08x], want: %v [%08x]", + f.Mode(), uint32(f.Mode()), match.Mode(), uint32(match.Mode())) + } + t.Logf("Mode is %v [%08x] for %s", f.Mode(), uint32(f.Mode()), f.Name) + + switch { + case f.FileInfo().IsDir(): + filebase := filepath.ToSlash(match.Name()) + if filepath.Base(f.Name) != filebase { + return fmt.Errorf("foldernames don't match got %v want %v", filepath.Base(f.Name), filebase) + } + if !strings.HasSuffix(f.Name, "/") { + return fmt.Errorf("foldernames must end with separator got %v", f.Name) + } + case f.Mode()&os.ModeSymlink != 0: + target, err := fs.Readlink(matchPath) + if err != nil { + return err + } + linkName, err := readZipFile(f) + if err != nil { + t.Fatal(err) + } + if target != string(linkName) { + return fmt.Errorf("symlink target does not match, got %s want %s", string(linkName), target) + } + default: + if uint64(match.Size()) != f.UncompressedSize64 { + return fmt.Errorf("size does not match got %v want %v", f.UncompressedSize64, match.Size()) + } + contentsFile, err := ioutil.ReadFile(matchPath) + if err != nil { + t.Fatal(err) + } + contentsZip, err := readZipFile(f) + if err != nil { + t.Fatal(err) + } + if string(contentsZip) != string(contentsFile) { + return fmt.Errorf("contents does not match, got %s want %s", contentsZip, contentsFile) + } + } + } + + if zipFiles != fileNumber { + return fmt.Errorf("not the same amount of files got %v want %v", zipFiles, fileNumber) + } + + return nil +}