diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 8bb81d390..b43c3cec9 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -17,13 +17,10 @@ import ( "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/fs" - "github.com/restic/restic/internal/hashing" "github.com/restic/restic/internal/pack" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui/progress" - "github.com/minio/sha256-simd" "golang.org/x/sync/errgroup" ) @@ -746,47 +743,6 @@ type Loader interface { Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error } -// DownloadAndHash is all-in-one helper to download content of the file at h to a temporary filesystem location -// and calculate ID of the contents. Returned (temporary) file is positioned at the beginning of the file; -// it is the reponsibility of the caller to close and delete the file. -func DownloadAndHash(ctx context.Context, be Loader, h restic.Handle) (tmpfile *os.File, hash restic.ID, size int64, err error) { - tmpfile, err = fs.TempFile("", "restic-temp-") - if err != nil { - return nil, restic.ID{}, -1, errors.Wrap(err, "TempFile") - } - - err = be.Load(ctx, h, 0, 0, func(rd io.Reader) (ierr error) { - _, ierr = tmpfile.Seek(0, io.SeekStart) - if ierr == nil { - ierr = tmpfile.Truncate(0) - } - if ierr != nil { - return ierr - } - hrd := hashing.NewReader(rd, sha256.New()) - size, ierr = io.Copy(tmpfile, hrd) - hash = restic.IDFromHash(hrd.Sum(nil)) - return ierr - }) - - if err != nil { - // ignore subsequent errors - _ = tmpfile.Close() - _ = os.Remove(tmpfile.Name()) - return nil, restic.ID{}, -1, errors.Wrap(err, "Load") - } - - _, err = tmpfile.Seek(0, io.SeekStart) - if err != nil { - // ignore subsequent errors - _ = tmpfile.Close() - _ = os.Remove(tmpfile.Name()) - return nil, restic.ID{}, -1, errors.Wrap(err, "Seek") - } - - return tmpfile, hash, size, err -} - type BackendLoadFn func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 22a424556..083b6edff 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -13,8 +13,6 @@ import ( "time" "github.com/restic/restic/internal/archiver" - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/fs" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -411,107 +409,3 @@ func TestRepositoryIncrementalIndex(t *testing.T) { } } } - -type backend struct { - rd io.Reader -} - -func (be backend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - return fn(be.rd) -} - -type retryBackend struct { - buf []byte -} - -func (be retryBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - err := fn(bytes.NewReader(be.buf[:len(be.buf)/2])) - if err != nil { - return err - } - - return fn(bytes.NewReader(be.buf)) -} - -func TestDownloadAndHash(t *testing.T) { - buf := make([]byte, 5*1024*1024+881) - _, err := io.ReadFull(rnd, buf) - if err != nil { - t.Fatal(err) - } - - var tests = []struct { - be repository.Loader - want []byte - }{ - { - be: backend{rd: bytes.NewReader(buf)}, - want: buf, - }, - { - be: retryBackend{buf: buf}, - want: buf, - }, - } - - for _, test := range tests { - t.Run("", func(t *testing.T) { - f, id, size, err := repository.DownloadAndHash(context.TODO(), test.be, restic.Handle{}) - if err != nil { - t.Error(err) - } - - want := restic.Hash(test.want) - if !want.Equal(id) { - t.Errorf("wrong hash returned, want %v, got %v", want.Str(), id.Str()) - } - - if size != int64(len(test.want)) { - t.Errorf("wrong size returned, want %v, got %v", test.want, size) - } - - err = f.Close() - if err != nil { - t.Error(err) - } - - err = fs.RemoveIfExists(f.Name()) - if err != nil { - t.Fatal(err) - } - }) - } -} - -type errorReader struct { - err error -} - -func (er errorReader) Read(p []byte) (n int, err error) { - return 0, er.err -} - -func TestDownloadAndHashErrors(t *testing.T) { - var tests = []struct { - be repository.Loader - err string - }{ - { - be: backend{rd: errorReader{errors.New("test error 1")}}, - err: "test error 1", - }, - } - - for _, test := range tests { - t.Run("", func(t *testing.T) { - _, _, _, err := repository.DownloadAndHash(context.TODO(), test.be, restic.Handle{}) - if err == nil { - t.Fatalf("wanted error %q, got nil", test.err) - } - - if errors.Cause(err).Error() != test.err { - t.Fatalf("wanted error %q, got %q", test.err, err) - } - }) - } -}