Merge pull request #1267 from harshavardhana/possible-fix-memory

Implement Size() and Len() to know the optimal size.
This commit is contained in:
Alexander Neumann 2017-09-23 14:04:15 +02:00
commit 429106340f
1 changed files with 58 additions and 3 deletions

View File

@ -4,7 +4,6 @@ import (
"context"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"strings"
@ -198,6 +197,53 @@ func (be *Backend) Path() string {
return be.cfg.Prefix
}
// nopCloserFile wraps *os.File and overwrites the Close() method with method
// that does nothing. In addition, the method Len() is implemented, which
// returns the size of the file (filesize - current offset).
type nopCloserFile struct {
*os.File
}
func (f nopCloserFile) Close() error {
debug.Log("prevented Close()")
return nil
}
// Len returns the remaining length of the file (filesize - current offset).
func (f nopCloserFile) Len() int {
debug.Log("Len() called")
fi, err := f.Stat()
if err != nil {
panic(err)
}
pos, err := f.Seek(0, io.SeekCurrent)
if err != nil {
panic(err)
}
size := fi.Size() - pos
debug.Log("returning file size %v", size)
return int(size)
}
type lenner interface {
Len() int
io.Reader
}
// nopCloserLenner wraps a lenner and overwrites the Close() method with method
// that does nothing. In addition, the method Size() is implemented, which
// returns the size of the file (filesize - current offset).
type nopCloserLenner struct {
lenner
}
func (f *nopCloserLenner) Close() error {
debug.Log("prevented Close()")
return nil
}
// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
debug.Log("Save %v", h)
@ -215,8 +261,17 @@ func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err
return errors.New("key already exists")
}
// prevent the HTTP client from closing a file
rd = ioutil.NopCloser(rd)
// FIXME: This is a workaround once we move to minio-go 4.0.x this can be
// removed and size can be directly provided.
if f, ok := rd.(*os.File); ok {
debug.Log("reader is %#T, using nopCloserFile{}", rd)
rd = nopCloserFile{f}
} else if l, ok := rd.(lenner); ok {
debug.Log("reader is %#T, using nopCloserLenner{}", rd)
rd = nopCloserLenner{l}
} else {
debug.Log("reader is %#T, no specific workaround enabled", rd)
}
be.sem.GetToken()
debug.Log("PutObject(%v, %v)", be.cfg.Bucket, objName)