rdelaage
/
restic
mirror of https://github.com/restic/restic.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3840 from greatroar/sftp-init 

Speed up restic init over slow SFTP links
This commit is contained in:
MichaelEischer 2022-07-31 19:54:59 +02:00 committed by GitHub
parent 4ffd479ba4 b9fa6e05bd
commit 846d021db5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 35 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
changelog/unreleased/issue-3837 Normal file
View File

@ -0,0 +1,8 @@
Enhancement: SFTP backend initialization is faster on slow links
Restic init on an SFTP backend now sends multiple mkdir commands to the
backend concurrently, to reduce the wait when creating a repository
over a very slow link.
https://github.com/restic/restic/issues/3837
https://github.com/restic/restic/pull/3840

2
go.mod
View File

@ -38,7 +38,7 @@ require (
golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064
golang.org/x/net v0.0.0-20220325170049-de3da57026de
golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
golang.org/x/sys v0.0.0-20220325203850-36772127a21f
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
golang.org/x/text v0.3.7

3
go.sum
View File

@ -439,8 +439,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

35
internal/backend/sftp/sftp.go
View File

@ -21,6 +21,7 @@ import (
"github.com/cenkalti/backoff/v4"
"github.com/pkg/sftp"
"golang.org/x/sync/errgroup"
)
// SFTP is a backend in a directory accessed via SFTP.
@ -116,8 +117,7 @@ func (r *SFTP) clientError() error {
}
// Open opens an sftp backend as described by the config by running
// "ssh" with the appropriate arguments (or cfg.Command, if set). The function
// preExec is run just before, postExec just after starting a program.
// "ssh" with the appropriate arguments (or cfg.Command, if set).
func Open(ctx context.Context, cfg Config) (*SFTP, error) {
debug.Log("open backend with config %#v", cfg)
@ -155,15 +155,29 @@ func Open(ctx context.Context, cfg Config) (*SFTP, error) {
return sftp, nil
}
func (r *SFTP) mkdirAllDataSubdirs() error {
func (r *SFTP) mkdirAllDataSubdirs(ctx context.Context, nconn uint) error {
// Run multiple MkdirAll calls concurrently. These involve multiple
// round-trips and we do a lot of them, so this whole operation can be slow
// on high-latency links.
g, _ := errgroup.WithContext(ctx)
// Use errgroup's built-in semaphore, because r.sem is not initialized yet.
g.SetLimit(int(nconn))
for _, d := range r.Paths() {
err := r.c.MkdirAll(d)
if err != nil {
return err
}
d := d
g.Go(func() error {
// First try Mkdir. For most directories in Paths, this takes one
// round trip, not counting duplicate parent creations causes by
// concurrency. MkdirAll first does Stat, then recursive MkdirAll
// on the parent, so calls typically take three round trips.
if err := r.c.Mkdir(d); err == nil {
return nil
}
return r.c.MkdirAll(d)
})
}
return nil
return g.Wait()
}
// Join combines path components with slashes (according to the sftp spec).
@ -214,8 +228,7 @@ func buildSSHCommand(cfg Config) (cmd string, args []string, err error) {
}
// Create creates an sftp backend as described by the config by running "ssh"
// with the appropriate arguments (or cfg.Command, if set). The function
// preExec is run just before, postExec just after starting a program.
// with the appropriate arguments (or cfg.Command, if set).
func Create(ctx context.Context, cfg Config) (*SFTP, error) {
cmd, args, err := buildSSHCommand(cfg)
if err != nil {
@ -242,7 +255,7 @@ func Create(ctx context.Context, cfg Config) (*SFTP, error) {
}
// create paths for data and refs
if err = sftp.mkdirAllDataSubdirs(); err != nil {
if err = sftp.mkdirAllDataSubdirs(ctx, cfg.Connections); err != nil {
return nil, err
}