complete RESITC_HOST environment handling & test

This commit adds support for specifying the `--host` option via the `RESTIC_HOST` environment variable. This is done by extending option processing in `cmd_backup.go` and for `restic.SnapshotFilter` in `find.go`.

.github/workflows/docker.yml

@@ -25,7 +25,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Log in to the Container registry
-        uses: docker/login-action@5139682d94efc37792e6b54386b5b470a68a4737
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -33,7 +33,7 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
@@ -45,7 +45,7 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226
+        uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20
 
       - name: Ensure consistent binaries
         run: |
@@ -55,7 +55,7 @@ jobs:
         if: github.ref != 'refs/heads/master'
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0
         with:
           push: true
           context: .

.github/workflows/tests.yml

@@ -74,7 +74,7 @@ jobs:
       - name: Get programs (Linux/macOS)
         run: |
           echo "build Go tools"
-          go install github.com/restic/rest-server/cmd/rest-server@latest
+          go install github.com/restic/rest-server/cmd/rest-server@master
 
           echo "install minio server"
           mkdir $HOME/bin
@@ -106,7 +106,7 @@ jobs:
           $ProgressPreference = 'SilentlyContinue'
 
           echo "build Go tools"
-          go install github.com/restic/rest-server/...
+          go install github.com/restic/rest-server/cmd/rest-server@master
 
           echo "install minio server"
           mkdir $Env:USERPROFILE/bin
@@ -247,6 +247,10 @@ jobs:
   lint:
     name: lint
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      # allow annotating code in the PR
+      checks: write
     steps:
       - name: Set up Go ${{ env.latest_go }}
         uses: actions/setup-go@v5
@@ -260,7 +264,7 @@ jobs:
         uses: golangci/golangci-lint-action@v4
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.56.1
+          version: v1.57.1
           args: --verbose --timeout 5m
 
         # only run golangci-lint for pull requests, otherwise ALL hints get
@@ -298,7 +302,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           # list of Docker images to use as base name for tags
           images: |
@@ -321,7 +325,7 @@ jobs:
 
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           push: false
           context: .

.golangci.yml

@@ -38,6 +38,8 @@ linters:
     # ensure that http response bodies are closed
     - bodyclose
 
+    - importas
+
 issues:
   # don't use the default exclude rules, this hides (among others) ignored
   # errors from Close() calls
@@ -58,4 +60,10 @@ issues:
   exclude-rules:
     # revive: ignore unused parameters in tests
     - path: (_test\.go|testing\.go|backend/.*/tests\.go)
-      text: "unused-parameter:"
+      text: "unused-parameter:"
+
+linters-settings:
+  importas:
+    alias:
+      - pkg: github.com/restic/restic/internal/test
+        alias: rtest

changelog/unreleased/issue-3600

@@ -0,0 +1,11 @@
+Bugfix: `backup` works if xattrs above the backup target cannot be read
+
+When backup targets are specified using absolute paths, then `backup` also
+includes information about the parent folders of the backup targets in the
+snapshot. If the extended attributes for some of these folders could not be
+read due to missing permissions, this caused the backup to fail. This has been
+fixed.
+
+https://github.com/restic/restic/issues/3600
+https://github.com/restic/restic/pull/4668
+https://forum.restic.net/t/parent-directories-above-the-snapshot-source-path-fatal-error-permission-denied/7216

changelog/unreleased/issue-4287

@@ -0,0 +1,14 @@
+Enhancement: support connection to rest-server using unix socket
+
+Restic now supports connecting to rest-server using a unix socket for
+rest-server version 0.13.0 or later.
+
+This allows running restic as follows:
+
+```
+rest-server --listen unix:/tmp/rest.socket --data /path/to/data &
+restic -r rest:http+unix:///tmp/rest.socket:/my_backup_repo/ [...]
+```
+
+https://github.com/restic/restic/issues/4287
+https://github.com/restic/restic/pull/4655

changelog/unreleased/issue-4602

@@ -0,0 +1,23 @@
+Change: Deprecate legacy index format and s3legacy layout
+
+Support for the legacy index format used by restic before version 0.2.0 has
+been deprecated and will be removed in the next minor restic version. You can
+use `restic repair index` to update the index to the current format.
+
+It is possible to temporarily reenable support for the legacy index format by
+setting the environment variable
+`RESTIC_FEATURES=deprecate-legacy-index=false`. Note that this feature flag
+will be removed in the next minor restic version.
+
+Support for the s3legacy layout used for the S3 backend before restic 0.7.0
+has been deprecated and will be removed in the next minor restic version. You
+can migrate your S3 repository using `RESTIC_FEATURES=deprecate-s3-legacy-layout=false restic migrate s3_layout`.
+
+It is possible to temporarily reenable support for the legacy s3layout by
+setting the environment variable
+`RESTIC_FEATURES=deprecate-s3-legacy-layout=false`. Note that this feature flag
+will be removed in the next minor restic version.
+
+https://github.com/restic/restic/issues/4602
+https://github.com/restic/restic/pull/4724
+https://github.com/restic/restic/pull/4743

changelog/unreleased/issue-4733

@@ -0,0 +1,10 @@
+Enhancement: Allow specifying `--host` via environment variable
+
+Restic commands that operate on snapshots, such as `restic backup` and
+`restic snapshots`, support the `--host` flag to specify the hostname for
+grouoping snapshots. They now permit selecting the hostname via the
+environment variable `RESTIC_HOST`. `--host` still takes precedence over the
+environment variable.
+
+https://github.com/restic/restic/issues/4733
+https://github.com/restic/restic/pull/4734

changelog/unreleased/issue-4744

@@ -0,0 +1,9 @@
+Change: Include full key ID in JSON output of `key list`
+
+We have changed the JSON output of the `key list` command to include the full
+key ID instead of just a shortened version, as the latter can be ambiguous
+in some rare cases. To derive the short ID, please truncate the full ID down to
+eight characters.
+
+https://github.com/restic/restic/issues/4744
+https://github.com/restic/restic/pull/4745

changelog/unreleased/issue-4760

@@ -0,0 +1,8 @@
+Bugfix: Fix possible error on concurrent cache cleanup
+
+If multiple restic processes concurrently cleaned up no longer existing files
+from the cache, this could cause some of the processes to fail with an `no such
+file or directory` error. This has been fixed.
+
+https://github.com/restic/restic/issues/4760
+https://github.com/restic/restic/pull/4761

changelog/unreleased/issue-693

@@ -0,0 +1,12 @@
+Enhancement: Support printing snapshot size in `snapshots` command
+
+The `snapshots` command now supports printing the snapshot size for snapshots
+created using this or a future restic version. For this, the `backup` command
+now stores the backup summary statistics in the snapshot.
+
+The text output of the `snapshots` command only shows the snapshot size. The
+other statistics are only included in the JSON output. To inspect these
+statistics use `restic snapshots --json` or `restic cat snapshot <snapshotID>`.
+
+https://github.com/restic/restic/issues/693
+https://github.com/restic/restic/pull/4705

changelog/unreleased/pull-4006

@@ -0,0 +1,16 @@
+Enhancement: (alpha) Store deviceID only for hardlinks
+
+Set `RESTIC_FEATURES=device-id-for-hardlinks` to enable this alpha feature.
+The feature flag will be removed after repository format version 3 becomes
+available or be replaced with a different solution.
+
+When creating backups from a filesystem snapshot, for example created using
+btrfs subvolumes, the deviceID of the filesystem changes compared to previous
+snapshots. This prevented restic from deduplicating the directory metadata of
+a snapshot.
+
+When this alpha feature is enabled, then the deviceID is only stored for
+hardlinks. This significantly reduces the metadata duplication for most
+backups.
+
+https://github.com/restic/restic/pull/4006

changelog/unreleased/pull-4503

@@ -4,4 +4,5 @@ If files on different devices had the same inode id, then the `stats` command
 did not correctly calculate the snapshot size. This has been fixed.
 
 https://github.com/restic/restic/pull/4503
+https://github.com/restic/restic/pull/4006
 https://forum.restic.net/t/possible-bug-in-stats/6461/8

changelog/unreleased/pull-4709

@@ -0,0 +1,10 @@
+Bugfix: Correct `--no-lock` handling of `ls` and `tag` command
+
+The `ls` command never locked the repository. This has been fixed. The old
+behavior is still supported using `ls --no-lock`. The latter invocation also
+works with older restic versions.
+
+The `tag` command erroneously accepted the `--no-lock` command. The command
+now always requires an exclusive lock.
+
+https://github.com/restic/restic/pull/4709

changelog/unreleased/pull-4737

@@ -0,0 +1,5 @@
+Enhancement: include snapshot id in reason field of forget JSON output
+
+The JSON output of the `forget` command now includes the `id` and `short_id` of a snapshot in the `reason` field.
+
+https://github.com/restic/restic/pull/4737

cmd/restic/cleanup.go

@@ -1,89 +1,41 @@
 package main
 
 import (
+	"context"
 	"os"
 	"os/signal"
-	"sync"
 	"syscall"
 
 	"github.com/restic/restic/internal/debug"
 )
 
-var cleanupHandlers struct {
-	sync.Mutex
-	list []func(code int) (int, error)
-	done bool
-	ch   chan os.Signal
+func createGlobalContext() context.Context {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	ch := make(chan os.Signal, 1)
+	go cleanupHandler(ch, cancel)
+	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
+
+	return ctx
 }
 
-func init() {
-	cleanupHandlers.ch = make(chan os.Signal, 1)
-	go CleanupHandler(cleanupHandlers.ch)
-	signal.Notify(cleanupHandlers.ch, syscall.SIGINT, syscall.SIGTERM)
-}
+// cleanupHandler handles the SIGINT and SIGTERM signals.
+func cleanupHandler(c <-chan os.Signal, cancel context.CancelFunc) {
+	s := <-c
+	debug.Log("signal %v received, cleaning up", s)
+	Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s)
 
-// AddCleanupHandler adds the function f to the list of cleanup handlers so
-// that it is executed when all the cleanup handlers are run, e.g. when SIGINT
-// is received.
-func AddCleanupHandler(f func(code int) (int, error)) {
-	cleanupHandlers.Lock()
-	defer cleanupHandlers.Unlock()
-
-	// reset the done flag for integration tests
-	cleanupHandlers.done = false
-
-	cleanupHandlers.list = append(cleanupHandlers.list, f)
-}
-
-// RunCleanupHandlers runs all registered cleanup handlers
-func RunCleanupHandlers(code int) int {
-	cleanupHandlers.Lock()
-	defer cleanupHandlers.Unlock()
-
-	if cleanupHandlers.done {
-		return code
+	if val, _ := os.LookupEnv("RESTIC_DEBUG_STACKTRACE_SIGINT"); val != "" {
+		_, _ = os.Stderr.WriteString("\n--- STACKTRACE START ---\n\n")
+		_, _ = os.Stderr.WriteString(debug.DumpStacktrace())
+		_, _ = os.Stderr.WriteString("\n--- STACKTRACE END ---\n")
 	}
-	cleanupHandlers.done = true
 
-	for _, f := range cleanupHandlers.list {
-		var err error
-		code, err = f(code)
-		if err != nil {
-			Warnf("error in cleanup handler: %v\n", err)
-		}
-	}
-	cleanupHandlers.list = nil
-	return code
+	cancel()
 }
 
-// CleanupHandler handles the SIGINT and SIGTERM signals.
-func CleanupHandler(c <-chan os.Signal) {
-	for s := range c {
-		debug.Log("signal %v received, cleaning up", s)
-		Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s)
-
-		if val, _ := os.LookupEnv("RESTIC_DEBUG_STACKTRACE_SIGINT"); val != "" {
-			_, _ = os.Stderr.WriteString("\n--- STACKTRACE START ---\n\n")
-			_, _ = os.Stderr.WriteString(debug.DumpStacktrace())
-			_, _ = os.Stderr.WriteString("\n--- STACKTRACE END ---\n")
-		}
-
-		code := 0
-
-		if s == syscall.SIGINT || s == syscall.SIGTERM {
-			code = 130
-		} else {
-			code = 1
-		}
-
-		Exit(code)
-	}
-}
-
-// Exit runs the cleanup handlers and then terminates the process with the
-// given exit code.
+// Exit terminates the process with the given exit code.
 func Exit(code int) {
-	code = RunCleanupHandlers(code)
 	debug.Log("exiting with status code %d", code)
 	os.Exit(code)
 }

cmd/restic/cmd_backup.go

@@ -114,7 +114,7 @@ func init() {
 	f.BoolVar(&backupOptions.StdinCommand, "stdin-from-command", false, "interpret arguments as command to execute and store its stdout")
 	f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)")
 	f.UintVar(&backupOptions.ReadConcurrency, "read-concurrency", 0, "read `n` files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)")
-	f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
+	f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually (default: $RESTIC_HOST). To prevent an expensive rescan use the \"parent\" flag")
 	f.StringVar(&backupOptions.Host, "hostname", "", "set the `hostname` for the snapshot manually")
 	err := f.MarkDeprecated("hostname", "use --host")
 	if err != nil {
@@ -137,6 +137,11 @@ func init() {
 	// parse read concurrency from env, on error the default value will be used
 	readConcurrency, _ := strconv.ParseUint(os.Getenv("RESTIC_READ_CONCURRENCY"), 10, 32)
 	backupOptions.ReadConcurrency = uint(readConcurrency)
+
+	// parse host from env, if not exists or empty the default value will be used
+	if host := os.Getenv("RESTIC_HOST"); host != "" {
+		backupOptions.Host = host
+	}
 }
 
 // filterExisting returns a slice of all existing items, or an error if no
@@ -451,6 +456,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 	}
 
 	timeStamp := time.Now()
+	backupStart := timeStamp
 	if opts.TimeStamp != "" {
 		timeStamp, err = time.ParseInLocation(TimeFormat, opts.TimeStamp, time.Local)
 		if err != nil {
@@ -462,10 +468,11 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 		Verbosef("open repository\n")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, opts.DryRun)
 	if err != nil {
 		return err
 	}
+	defer unlock()
 
 	var progressPrinter backup.ProgressPrinter
 	if gopts.JSON {
@@ -477,22 +484,6 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 		calculateProgressInterval(!gopts.Quiet, gopts.JSON))
 	defer progressReporter.Done()
 
-	if opts.DryRun {
-		repo.SetDryRun()
-	}
-
-	if !gopts.JSON {
-		progressPrinter.V("lock repository")
-	}
-	if !opts.DryRun {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
-
 	// rejectByNameFuncs collect functions that can reject items from the backup based on path only
 	rejectByNameFuncs, err := collectRejectByNameFuncs(opts, repo)
 	if err != nil {
@@ -640,6 +631,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 	snapshotOpts := archiver.SnapshotOptions{
 		Excludes:       opts.Excludes,
 		Tags:           opts.Tags.Flatten(),
+		BackupStart:    backupStart,
 		Time:           timeStamp,
 		Hostname:       opts.Host,
 		ParentSnapshot: parentSnapshot,
@@ -649,7 +641,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 	if !gopts.JSON {
 		progressPrinter.V("start backup on %v", targets)
 	}
-	_, id, err := arch.Snapshot(ctx, targets, snapshotOpts)
+	_, id, summary, err := arch.Snapshot(ctx, targets, snapshotOpts)
 
 	// cleanly shutdown all running goroutines
 	cancel()
@@ -663,7 +655,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 	}
 
 	// Report finished execution
-	progressReporter.Finish(id, opts.DryRun)
+	progressReporter.Finish(id, summary, opts.DryRun)
 	if !gopts.JSON && !opts.DryRun {
 		progressPrinter.P("snapshot %s saved\n", id.Str())
 	}

cmd/restic/cmd_backup_integration_test.go

@@ -9,7 +9,6 @@ import (
 	"runtime"
 	"testing"
 
-	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
@@ -250,29 +249,18 @@ func TestBackupTreeLoadError(t *testing.T) {
 	opts := BackupOptions{}
 	// Backup a subdirectory first, such that we can remove the tree pack for the subdirectory
 	testRunBackup(t, env.testdata, []string{"test"}, opts, env.gopts)
-
-	r, err := OpenRepository(context.TODO(), env.gopts)
-	rtest.OK(t, err)
-	rtest.OK(t, r.LoadIndex(context.TODO(), nil))
-	treePacks := restic.NewIDSet()
-	r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
-		if pb.Type == restic.TreeBlob {
-			treePacks.Insert(pb.PackID)
-		}
-	})
+	treePacks := listTreePacks(env.gopts, t)
 
 	testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
 	testRunCheck(t, env.gopts)
 
 	// delete the subdirectory pack first
-	for id := range treePacks {
-		rtest.OK(t, r.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()}))
-	}
+	removePacks(env.gopts, t, treePacks)
 	testRunRebuildIndex(t, env.gopts)
 	// now the repo is missing the tree blob in the index; check should report this
 	testRunCheckMustFail(t, env.gopts)
 	// second backup should report an error but "heal" this situation
-	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
+	err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
 	rtest.Assert(t, err != nil, "backup should have reported an error for the subdirectory")
 	testRunCheck(t, env.gopts)
 
@@ -406,6 +394,7 @@ func TestIncrementalBackup(t *testing.T) {
 	t.Logf("repository grown by %d bytes", stat3.size-stat2.size)
 }
 
+// nolint: staticcheck // false positive nil pointer dereference check
 func TestBackupTags(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
@@ -441,6 +430,7 @@ func TestBackupTags(t *testing.T) {
 		"expected parent to be %v, got %v", parent.ID, newest.Parent)
 }
 
+// nolint: staticcheck // false positive nil pointer dereference check
 func TestBackupProgramVersion(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()

cmd/restic/cmd_cat.go

@@ -64,19 +64,11 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
-
-	if !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
+	defer unlock()
 
 	tpe := args[0]
 

cmd/restic/cmd_check.go

@@ -199,25 +199,16 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	}
 
 	cleanup := prepareCheckCache(opts, &gopts)
-	AddCleanupHandler(func(code int) (int, error) {
-		cleanup()
-		return code, nil
-	})
-
-	repo, err := OpenRepository(ctx, gopts)
-	if err != nil {
-		return err
-	}
+	defer cleanup()
 
 	if !gopts.NoLock {
 		Verbosef("create exclusive lock for repository\n")
-		var lock *restic.Lock
-		lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
 	}
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, gopts.NoLock)
+	if err != nil {
+		return err
+	}
+	defer unlock()
 
 	chkr := checker.New(repo, opts.CheckUnused)
 	err = chkr.LoadSnapshots(ctx)
@@ -228,15 +219,23 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	Verbosef("load indexes\n")
 	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
 	hints, errs := chkr.LoadIndex(ctx, bar)
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 
 	errorsFound := false
 	suggestIndexRebuild := false
+	suggestLegacyIndexRebuild := false
 	mixedFound := false
 	for _, hint := range hints {
 		switch hint.(type) {
-		case *checker.ErrDuplicatePacks, *checker.ErrOldIndexFormat:
+		case *checker.ErrDuplicatePacks:
 			Printf("%v\n", hint)
 			suggestIndexRebuild = true
+		case *checker.ErrOldIndexFormat:
+			Warnf("error: %v\n", hint)
+			suggestLegacyIndexRebuild = true
+			errorsFound = true
 		case *checker.ErrMixedPack:
 			Printf("%v\n", hint)
 			mixedFound = true
@@ -247,7 +246,10 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	}
 
 	if suggestIndexRebuild {
-		Printf("Duplicate packs/old indexes are non-critical, you can run `restic repair index' to correct this.\n")
+		Printf("Duplicate packs are non-critical, you can run `restic repair index' to correct this.\n")
+	}
+	if suggestLegacyIndexRebuild {
+		Warnf("Found indexes using the legacy format, you must run `restic repair index' to correct this.\n")
 	}
 	if mixedFound {
 		Printf("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
@@ -281,6 +283,9 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	if orphanedPacks > 0 {
 		Verbosef("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks)
 	}
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 
 	Verbosef("check snapshots, trees and blobs\n")
 	errChan = make(chan error)
@@ -314,9 +319,16 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	// Must happen after `errChan` is read from in the above loop to avoid
 	// deadlocking in the case of errors.
 	wg.Wait()
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 
 	if opts.CheckUnused {
-		for _, id := range chkr.UnusedBlobs(ctx) {
+		unused, err := chkr.UnusedBlobs(ctx)
+		if err != nil {
+			return err
+		}
+		for _, id := range unused {
 			Verbosef("unused blob %v\n", id)
 			errorsFound = true
 		}
@@ -393,10 +405,13 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 		doReadData(packs)
 	}
 
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+
 	if errorsFound {
 		return errors.Fatal("repository contains errors")
 	}
-
 	Verbosef("no errors were found\n")
 
 	return nil

cmd/restic/cmd_copy.go

@@ -53,7 +53,7 @@ func init() {
 }
 
 func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error {
-	secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination")
+	secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(ctx, opts.secondaryRepoOptions, gopts, "destination")
 	if err != nil {
 		return err
 	}
@@ -62,30 +62,17 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
 		gopts, secondaryGopts = secondaryGopts, gopts
 	}
 
-	srcRepo, err := OpenRepository(ctx, gopts)
+	ctx, srcRepo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
+	defer unlock()
 
-	dstRepo, err := OpenRepository(ctx, secondaryGopts)
-	if err != nil {
-		return err
-	}
-
-	if !gopts.NoLock {
-		var srcLock *restic.Lock
-		srcLock, ctx, err = lockRepo(ctx, srcRepo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(srcLock)
-		if err != nil {
-			return err
-		}
-	}
-
-	dstLock, ctx, err := lockRepo(ctx, dstRepo, gopts.RetryLock, gopts.JSON)
-	defer unlockRepo(dstLock)
+	ctx, dstRepo, unlock, err := openWithAppendLock(ctx, secondaryGopts, false)
 	if err != nil {
 		return err
 	}
+	defer unlock()
 
 	srcSnapshotLister, err := restic.MemorizeList(ctx, srcRepo, restic.SnapshotFile)
 	if err != nil {
@@ -116,6 +103,9 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
 		// also consider identical snapshot copies
 		dstSnapshotByOriginal[*sn.ID()] = append(dstSnapshotByOriginal[*sn.ID()], sn)
 	}
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 
 	// remember already processed trees across all snapshots
 	visitedTrees := restic.NewIDSet()
@@ -160,7 +150,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
 		}
 		Verbosef("snapshot %s saved\n", newID.Str())
 	}
-	return nil
+	return ctx.Err()
 }
 
 func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool {

cmd/restic/cmd_debug.go

@@ -153,19 +153,11 @@ func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error
 		return errors.Fatal("type not specified")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
-
-	if !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
+	defer unlock()
 
 	tpe := args[0]
 
@@ -442,10 +434,15 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
 }
 
 func runDebugExamine(ctx context.Context, gopts GlobalOptions, opts DebugExamineOptions, args []string) error {
-	repo, err := OpenRepository(ctx, gopts)
+	if opts.ExtractPack && gopts.NoLock {
+		return fmt.Errorf("--extract-pack and --no-lock are mutually exclusive")
+	}
+
+	ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
+	defer unlock()
 
 	ids := make([]restic.ID, 0)
 	for _, name := range args {
@@ -464,15 +461,6 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, opts DebugExamine
 		return errors.Fatal("no pack files to examine")
 	}
 
-	if !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
-
 	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
 	err = repo.LoadIndex(ctx, bar)
 	if err != nil {

cmd/restic/cmd_diff.go

@@ -344,19 +344,11 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 		return errors.Fatalf("specify two snapshot IDs")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
-
-	if !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
+	defer unlock()
 
 	// cache snapshots listing
 	be, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)

cmd/restic/cmd_dump.go

@@ -131,19 +131,11 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
 
 	splittedPath := splitPath(path.Clean(pathToPrint))
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
-
-	if !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
+	defer unlock()
 
 	sn, subfolder, err := (&restic.SnapshotFilter{
 		Hosts: opts.Hosts,

cmd/restic/cmd_find.go

@@ -439,7 +439,10 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
 
 	if err != errAllPacksFound {
 		// try to resolve unknown pack ids from the index
-		packIDs = f.indexPacksToBlobs(ctx, packIDs)
+		packIDs, err = f.indexPacksToBlobs(ctx, packIDs)
+		if err != nil {
+			return err
+		}
 	}
 
 	if len(packIDs) > 0 {
@@ -456,13 +459,13 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
 	return nil
 }
 
-func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struct{}) map[string]struct{} {
+func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struct{}) (map[string]struct{}, error) {
 	wctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
 	// remember which packs were found in the index
 	indexPackIDs := make(map[string]struct{})
-	f.repo.Index().Each(wctx, func(pb restic.PackedBlob) {
+	err := f.repo.Index().Each(wctx, func(pb restic.PackedBlob) {
 		idStr := pb.PackID.String()
 		// keep entry in packIDs as Each() returns individual index entries
 		matchingID := false
@@ -481,6 +484,9 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
 			indexPackIDs[idStr] = struct{}{}
 		}
 	})
+	if err != nil {
+		return nil, err
+	}
 
 	for id := range indexPackIDs {
 		delete(packIDs, id)
@@ -493,7 +499,7 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
 		}
 		Warnf("some pack files are missing from the repository, getting their blobs from the repository index: %v\n\n", list)
 	}
-	return packIDs
+	return packIDs, nil
 }
 
 func (f *Finder) findObjectPack(id string, t restic.BlobType) {
@@ -563,19 +569,11 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
 		return errors.Fatal("cannot have several ID types")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
-
-	if !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
+	defer unlock()
 
 	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
@@ -616,6 +614,9 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
 	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, opts.Snapshots) {
 		filteredSnapshots = append(filteredSnapshots, sn)
 	}
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 
 	sort.Slice(filteredSnapshots, func(i, j int) bool {
 		return filteredSnapshots[i].Time.Before(filteredSnapshots[j].Time)

cmd/restic/cmd_forget.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui/termstatus"
 	"github.com/spf13/cobra"
 )
 
@@ -33,7 +34,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runForget(cmd.Context(), forgetOptions, forgetPruneOptions, globalOptions, args)
+		term, cancel := setupTermstatus()
+		defer cancel()
+		return runForget(cmd.Context(), forgetOptions, forgetPruneOptions, globalOptions, term, args)
 	},
 }
 
@@ -152,7 +155,7 @@ func verifyForgetOptions(opts *ForgetOptions) error {
 	return nil
 }
 
-func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOptions, gopts GlobalOptions, args []string) error {
+func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
 	err := verifyForgetOptions(&opts)
 	if err != nil {
 		return err
@@ -163,23 +166,21 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
 		return err
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
-	if err != nil {
-		return err
-	}
-
 	if gopts.NoLock && !opts.DryRun {
 		return errors.Fatal("--no-lock is only applicable in combination with --dry-run for forget command")
 	}
 
-	if !opts.DryRun || !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun && gopts.NoLock)
+	if err != nil {
+		return err
 	}
+	defer unlock()
+
+	verbosity := gopts.verbosity
+	if gopts.JSON {
+		verbosity = 0
+	}
+	printer := newTerminalProgressPrinter(verbosity, term)
 
 	var snapshots restic.Snapshots
 	removeSnIDs := restic.NewIDSet()
@@ -187,6 +188,9 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
 	for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
 		snapshots = append(snapshots, sn)
 	}
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 
 	var jsonGroups []*ForgetGroup
 
@@ -218,15 +222,11 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
 		}
 
 		if policy.Empty() && len(args) == 0 {
-			if !gopts.JSON {
-				Verbosef("no policy was specified, no snapshots will be removed\n")
-			}
+			printer.P("no policy was specified, no snapshots will be removed\n")
 		}
 
 		if !policy.Empty() {
-			if !gopts.JSON {
-				Verbosef("Applying Policy: %v\n", policy)
-			}
+			printer.P("Applying Policy: %v\n", policy)
 
 			for k, snapshotGroup := range snapshotGroups {
 				if gopts.Verbose >= 1 && !gopts.JSON {
@@ -249,20 +249,20 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
 				keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy)
 
 				if len(keep) != 0 && !gopts.Quiet && !gopts.JSON {
-					Printf("keep %d snapshots:\n", len(keep))
+					printer.P("keep %d snapshots:\n", len(keep))
 					PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact)
-					Printf("\n")
+					printer.P("\n")
 				}
-				addJSONSnapshots(&fg.Keep, keep)
+				fg.Keep = asJSONSnapshots(keep)
 
 				if len(remove) != 0 && !gopts.Quiet && !gopts.JSON {
-					Printf("remove %d snapshots:\n", len(remove))
+					printer.P("remove %d snapshots:\n", len(remove))
 					PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact)
-					Printf("\n")
+					printer.P("\n")
 				}
-				addJSONSnapshots(&fg.Remove, remove)
+				fg.Remove = asJSONSnapshots(remove)
 
-				fg.Reasons = reasons
+				fg.Reasons = asJSONKeeps(reasons)
 
 				jsonGroups = append(jsonGroups, &fg)
 
@@ -273,16 +273,27 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
 		}
 	}
 
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+
 	if len(removeSnIDs) > 0 {
 		if !opts.DryRun {
-			err := DeleteFilesChecked(ctx, gopts, repo, removeSnIDs, restic.SnapshotFile)
+			bar := printer.NewCounter("files deleted")
+			err := restic.ParallelRemove(ctx, repo, removeSnIDs, restic.SnapshotFile, func(id restic.ID, err error) error {
+				if err != nil {
+					printer.E("unable to remove %v/%v from the repository\n", restic.SnapshotFile, id)
+				} else {
+					printer.VV("removed %v/%v\n", restic.SnapshotFile, id)
+				}
+				return nil
+			}, bar)
+			bar.Done()
 			if err != nil {
 				return err
 			}
 		} else {
-			if !gopts.JSON {
-				Printf("Would have removed the following snapshots:\n%v\n\n", removeSnIDs)
-			}
+			printer.P("Would have removed the following snapshots:\n%v\n\n", removeSnIDs)
 		}
 	}
 
@@ -294,15 +305,13 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
 	}
 
 	if len(removeSnIDs) > 0 && opts.Prune {
-		if !gopts.JSON {
-			if opts.DryRun {
-				Verbosef("%d snapshots would be removed, running prune dry run\n", len(removeSnIDs))
-			} else {
-				Verbosef("%d snapshots have been removed, running prune\n", len(removeSnIDs))
-			}
+		if opts.DryRun {
+			printer.P("%d snapshots would be removed, running prune dry run\n", len(removeSnIDs))
+		} else {
+			printer.P("%d snapshots have been removed, running prune\n", len(removeSnIDs))
 		}
 		pruneOptions.DryRun = opts.DryRun
-		return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs)
+		return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs, term)
 	}
 
 	return nil
@@ -310,23 +319,47 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
 
 // ForgetGroup helps to print what is forgotten in JSON.
 type ForgetGroup struct {
-	Tags    []string            `json:"tags"`
-	Host    string              `json:"host"`
-	Paths   []string            `json:"paths"`
-	Keep    []Snapshot          `json:"keep"`
-	Remove  []Snapshot          `json:"remove"`
-	Reasons []restic.KeepReason `json:"reasons"`
+	Tags    []string     `json:"tags"`
+	Host    string       `json:"host"`
+	Paths   []string     `json:"paths"`
+	Keep    []Snapshot   `json:"keep"`
+	Remove  []Snapshot   `json:"remove"`
+	Reasons []KeepReason `json:"reasons"`
 }
 
-func addJSONSnapshots(js *[]Snapshot, list restic.Snapshots) {
+func asJSONSnapshots(list restic.Snapshots) []Snapshot {
+	var resultList []Snapshot
 	for _, sn := range list {
 		k := Snapshot{
 			Snapshot: sn,
 			ID:       sn.ID(),
 			ShortID:  sn.ID().Str(),
 		}
-		*js = append(*js, k)
+		resultList = append(resultList, k)
 	}
+	return resultList
+}
+
+// KeepReason helps to print KeepReasons as JSON with Snapshots with their ID included.
+type KeepReason struct {
+	Snapshot Snapshot `json:"snapshot"`
+	Matches  []string `json:"matches"`
+}
+
+func asJSONKeeps(list []restic.KeepReason) []KeepReason {
+	var resultList []KeepReason
+	for _, keep := range list {
+		k := KeepReason{
+			Snapshot: Snapshot{
+				Snapshot: keep.Snapshot,
+				ID:       keep.Snapshot.ID(),
+				ShortID:  keep.Snapshot.ID().Str(),
+			},
+			Matches: keep.Matches,
+		}
+		resultList = append(resultList, k)
+	}
+	return resultList
 }
 
 func printJSONForget(stdout io.Writer, forgets []*ForgetGroup) error {

cmd/restic/cmd_forget_integration_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	rtest "github.com/restic/restic/internal/test"
+	"github.com/restic/restic/internal/ui/termstatus"
 )
 
 func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
@@ -12,5 +13,7 @@ func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
 	pruneOpts := PruneOptions{
 		MaxUnused: "5%",
 	}
-	rtest.OK(t, runForget(context.TODO(), opts, pruneOpts, gopts, args))
+	rtest.OK(t, withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+		return runForget(context.TODO(), opts, pruneOpts, gopts, term, args)
+	}))
 }

cmd/restic/cmd_init.go

@@ -80,7 +80,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
 		return err
 	}
 
-	gopts.password, err = ReadPasswordTwice(gopts,
+	gopts.password, err = ReadPasswordTwice(ctx, gopts,
 		"enter password for new repository: ",
 		"enter password again: ")
 	if err != nil {
@@ -131,7 +131,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
 
 func maybeReadChunkerPolynomial(ctx context.Context, opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) {
 	if opts.CopyChunkerParameters {
-		otherGopts, _, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary")
+		otherGopts, _, err := fillSecondaryGlobalOpts(ctx, opts.secondaryRepoOptions, gopts, "secondary")
 		if err != nil {
 			return nil, err
 		}

cmd/restic/cmd_key_add.go

@@ -50,22 +50,17 @@ func runKeyAdd(ctx context.Context, gopts GlobalOptions, opts KeyAddOptions, arg
 		return fmt.Errorf("the key add command expects no arguments, only options - please see `restic help key add` for usage and flags")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
-	if err != nil {
-		return err
-	}
-
-	lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-	defer unlockRepo(lock)
+	ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, false)
 	if err != nil {
 		return err
 	}
+	defer unlock()
 
 	return addKey(ctx, repo, gopts, opts)
 }
 
 func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyAddOptions) error {
-	pw, err := getNewPassword(gopts, opts.NewPasswordFile)
+	pw, err := getNewPassword(ctx, gopts, opts.NewPasswordFile)
 	if err != nil {
 		return err
 	}
@@ -88,7 +83,7 @@ func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOption
 // testKeyNewPassword is used to set a new password during integration testing.
 var testKeyNewPassword string
 
-func getNewPassword(gopts GlobalOptions, newPasswordFile string) (string, error) {
+func getNewPassword(ctx context.Context, gopts GlobalOptions, newPasswordFile string) (string, error) {
 	if testKeyNewPassword != "" {
 		return testKeyNewPassword, nil
 	}
@@ -102,7 +97,7 @@ func getNewPassword(gopts GlobalOptions, newPasswordFile string) (string, error)
 	newopts := gopts
 	newopts.password = ""
 
-	return ReadPasswordTwice(newopts,
+	return ReadPasswordTwice(ctx, newopts,
 		"enter new password: ",
 		"enter password again: ")
 }

cmd/restic/cmd_key_list.go

@@ -40,19 +40,11 @@ func runKeyList(ctx context.Context, gopts GlobalOptions, args []string) error {
 		return fmt.Errorf("the key list command expects no arguments, only options - please see `restic help key list` for usage and flags")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
-
-	if !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
+	defer unlock()
 
 	return listKeys(ctx, repo, gopts)
 }
@@ -61,6 +53,7 @@ func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions
 	type keyInfo struct {
 		Current  bool   `json:"current"`
 		ID       string `json:"id"`
+		ShortID  string `json:"-"`
 		UserName string `json:"userName"`
 		HostName string `json:"hostName"`
 		Created  string `json:"created"`
@@ -78,7 +71,8 @@ func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions
 
 		key := keyInfo{
 			Current:  id == s.KeyID(),
-			ID:       id.Str(),
+			ID:       id.String(),
+			ShortID:  id.Str(),
 			UserName: k.Username,
 			HostName: k.Hostname,
 			Created:  k.Created.Local().Format(TimeFormat),
@@ -99,7 +93,7 @@ func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions
 	}
 
 	tab := table.New()
-	tab.AddColumn(" ID", "{{if .Current}}*{{else}} {{end}}{{ .ID }}")
+	tab.AddColumn(" ID", "{{if .Current}}*{{else}} {{end}}{{ .ShortID }}")
 	tab.AddColumn("User", "{{ .UserName }}")
 	tab.AddColumn("Host", "{{ .HostName }}")
 	tab.AddColumn("Created", "{{ .Created }}")

cmd/restic/cmd_key_passwd.go

@@ -47,22 +47,17 @@ func runKeyPasswd(ctx context.Context, gopts GlobalOptions, opts KeyPasswdOption
 		return fmt.Errorf("the key passwd command expects no arguments, only options - please see `restic help key passwd` for usage and flags")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
-	if err != nil {
-		return err
-	}
-
-	lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-	defer unlockRepo(lock)
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
 	if err != nil {
 		return err
 	}
+	defer unlock()
 
 	return changePassword(ctx, repo, gopts, opts)
 }
 
 func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyPasswdOptions) error {
-	pw, err := getNewPassword(gopts, opts.NewPasswordFile)
+	pw, err := getNewPassword(ctx, gopts, opts.NewPasswordFile)
 	if err != nil {
 		return err
 	}

cmd/restic/cmd_key_remove.go

@@ -37,20 +37,13 @@ func runKeyRemove(ctx context.Context, gopts GlobalOptions, args []string) error
 		return fmt.Errorf("key remove expects one argument as the key id")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
 	if err != nil {
 		return err
 	}
+	defer unlock()
 
-	lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-	defer unlockRepo(lock)
-	if err != nil {
-		return err
-	}
-
-	idPrefix := args[0]
-
-	return deleteKey(ctx, repo, idPrefix)
+	return deleteKey(ctx, repo, args[0])
 }
 
 func deleteKey(ctx context.Context, repo *repository.Repository, idPrefix string) error {

cmd/restic/cmd_list.go

@@ -36,19 +36,11 @@ func runList(ctx context.Context, gopts GlobalOptions, args []string) error {
 		return errors.Fatal("type not specified")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock || args[0] == "locks")
 	if err != nil {
 		return err
 	}
-
-	if !gopts.NoLock && args[0] != "locks" {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
+	defer unlock()
 
 	var t restic.FileType
 	switch args[0] {
@@ -67,10 +59,9 @@ func runList(ctx context.Context, gopts GlobalOptions, args []string) error {
 			if err != nil {
 				return err
 			}
-			idx.Each(ctx, func(blobs restic.PackedBlob) {
+			return idx.Each(ctx, func(blobs restic.PackedBlob) {
 				Printf("%v %v\n", blobs.Type, blobs.ID)
 			})
-			return nil
 		})
 	default:
 		return errors.Fatal("invalid type")

cmd/restic/cmd_ls.go

@@ -309,10 +309,11 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		return false
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
+	defer unlock()
 
 	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {

cmd/restic/cmd_migrate.go

@@ -117,16 +117,11 @@ func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptio
 }
 
 func runMigrate(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, args []string) error {
-	repo, err := OpenRepository(ctx, gopts)
-	if err != nil {
-		return err
-	}
-
-	lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-	defer unlockRepo(lock)
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
 	if err != nil {
 		return err
 	}
+	defer unlock()
 
 	if len(args) == 0 {
 		return checkMigrations(ctx, repo)

cmd/restic/cmd_mount.go

@@ -125,19 +125,11 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
 	debug.Log("start mount")
 	defer debug.Log("finish mount")
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
-
-	if !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
+	defer unlock()
 
 	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
 	err = repo.LoadIndex(ctx, bar)
@@ -160,28 +152,15 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
 		}
 	}
 
-	AddCleanupHandler(func(code int) (int, error) {
-		debug.Log("running umount cleanup handler for mount at %v", mountpoint)
-		err := umount(mountpoint)
-		if err != nil {
-			Warnf("unable to umount (maybe already umounted or still in use?): %v\n", err)
-		}
-		// replace error code of sigint
-		if code == 130 {
-			code = 0
-		}
-		return code, nil
-	})
+	systemFuse.Debug = func(msg interface{}) {
+		debug.Log("fuse: %v", msg)
+	}
 
 	c, err := systemFuse.Mount(mountpoint, mountOptions...)
 	if err != nil {
 		return err
 	}
 
-	systemFuse.Debug = func(msg interface{}) {
-		debug.Log("fuse: %v", msg)
-	}
-
 	cfg := fuse.Config{
 		OwnerIsRoot:   opts.OwnerRoot,
 		Filter:        opts.SnapshotFilter,
@@ -195,15 +174,26 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
 	Printf("When finished, quit with Ctrl-c here or umount the mountpoint.\n")
 
 	debug.Log("serving mount at %v", mountpoint)
-	err = fs.Serve(c, root)
-	if err != nil {
-		return err
+
+	done := make(chan struct{})
+
+	go func() {
+		defer close(done)
+		err = fs.Serve(c, root)
+	}()
+
+	select {
+	case <-ctx.Done():
+		debug.Log("running umount cleanup handler for mount at %v", mountpoint)
+		err := systemFuse.Unmount(mountpoint)
+		if err != nil {
+			Warnf("unable to umount (maybe already umounted or still in use?): %v\n", err)
+		}
+
+		return ErrOK
+	case <-done:
+		// clean shutdown, nothing to do
 	}
 
-	<-c.Ready
-	return c.MountError
-}
-
-func umount(mountpoint string) error {
-	return systemFuse.Unmount(mountpoint)
+	return err
 }

cmd/restic/cmd_mount_integration_test.go

@@ -12,7 +12,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/restic/restic/internal/repository"
+	systemFuse "github.com/anacrolix/fuse"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )
@@ -66,7 +66,7 @@ func testRunMount(t testing.TB, gopts GlobalOptions, dir string, wg *sync.WaitGr
 func testRunUmount(t testing.TB, dir string) {
 	var err error
 	for i := 0; i < mountWait; i++ {
-		if err = umount(dir); err == nil {
+		if err = systemFuse.Unmount(dir); err == nil {
 			t.Logf("directory %v umounted", dir)
 			return
 		}
@@ -86,12 +86,12 @@ func listSnapshots(t testing.TB, dir string) []string {
 	return names
 }
 
-func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Repository, mountpoint, repodir string, snapshotIDs restic.IDs, expectedSnapshotsInFuseDir int) {
+func checkSnapshots(t testing.TB, gopts GlobalOptions, mountpoint string, snapshotIDs restic.IDs, expectedSnapshotsInFuseDir int) {
 	t.Logf("checking for %d snapshots: %v", len(snapshotIDs), snapshotIDs)
 
 	var wg sync.WaitGroup
 	wg.Add(1)
-	go testRunMount(t, global, mountpoint, &wg)
+	go testRunMount(t, gopts, mountpoint, &wg)
 	waitForMount(t, mountpoint)
 	defer wg.Wait()
 	defer testRunUmount(t, mountpoint)
@@ -100,7 +100,7 @@ func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Reposit
 		t.Fatal(`virtual directory "snapshots" doesn't exist`)
 	}
 
-	ids := listSnapshots(t, repodir)
+	ids := listSnapshots(t, gopts.Repo)
 	t.Logf("found %v snapshots in repo: %v", len(ids), ids)
 
 	namesInSnapshots := listSnapshots(t, mountpoint)
@@ -124,6 +124,10 @@ func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Reposit
 		}
 	}
 
+	_, repo, unlock, err := openWithReadLock(context.TODO(), gopts, false)
+	rtest.OK(t, err)
+	defer unlock()
+
 	for _, id := range snapshotIDs {
 		snapshot, err := restic.LoadSnapshot(context.TODO(), repo, id)
 		rtest.OK(t, err)
@@ -166,10 +170,7 @@ func TestMount(t *testing.T) {
 
 	testRunInit(t, env.gopts)
 
-	repo, err := OpenRepository(context.TODO(), env.gopts)
-	rtest.OK(t, err)
-
-	checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, []restic.ID{}, 0)
+	checkSnapshots(t, env.gopts, env.mountpoint, []restic.ID{}, 0)
 
 	rtest.SetupTarTestFixture(t, env.testdata, filepath.Join("testdata", "backup-data.tar.gz"))
 
@@ -179,7 +180,7 @@ func TestMount(t *testing.T) {
 	rtest.Assert(t, len(snapshotIDs) == 1,
 		"expected one snapshot, got %v", snapshotIDs)
 
-	checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, snapshotIDs, 2)
+	checkSnapshots(t, env.gopts, env.mountpoint, snapshotIDs, 2)
 
 	// second backup, implicit incremental
 	testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts)
@@ -187,7 +188,7 @@ func TestMount(t *testing.T) {
 	rtest.Assert(t, len(snapshotIDs) == 2,
 		"expected two snapshots, got %v", snapshotIDs)
 
-	checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, snapshotIDs, 3)
+	checkSnapshots(t, env.gopts, env.mountpoint, snapshotIDs, 3)
 
 	// third backup, explicit incremental
 	bopts := BackupOptions{Parent: snapshotIDs[0].String()}
@@ -196,7 +197,7 @@ func TestMount(t *testing.T) {
 	rtest.Assert(t, len(snapshotIDs) == 3,
 		"expected three snapshots, got %v", snapshotIDs)
 
-	checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, snapshotIDs, 4)
+	checkSnapshots(t, env.gopts, env.mountpoint, snapshotIDs, 4)
 }
 
 func TestMountSameTimestamps(t *testing.T) {
@@ -211,14 +212,11 @@ func TestMountSameTimestamps(t *testing.T) {
 
 	rtest.SetupTarTestFixture(t, env.base, filepath.Join("testdata", "repo-same-timestamps.tar.gz"))
 
-	repo, err := OpenRepository(context.TODO(), env.gopts)
-	rtest.OK(t, err)
-
 	ids := []restic.ID{
 		restic.TestParseID("280303689e5027328889a06d718b729e96a1ce6ae9ef8290bff550459ae611ee"),
 		restic.TestParseID("75ad6cdc0868e082f2596d5ab8705e9f7d87316f5bf5690385eeff8dbe49d9f5"),
 		restic.TestParseID("5fd0d8b2ef0fa5d23e58f1e460188abb0f525c0f0c4af8365a1280c807a80a1b"),
 	}
 
-	checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, ids, 4)
+	checkSnapshots(t, env.gopts, env.mountpoint, ids, 4)
 }

cmd/restic/cmd_prune.go

@@ -4,26 +4,20 @@ import (
 	"context"
 	"math"
 	"runtime"
-	"sort"
 	"strconv"
 	"strings"
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/index"
-	"github.com/restic/restic/internal/pack"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/ui"
 	"github.com/restic/restic/internal/ui/progress"
+	"github.com/restic/restic/internal/ui/termstatus"
 
 	"github.com/spf13/cobra"
 )
 
-var errorIndexIncomplete = errors.Fatal("index is not complete")
-var errorPacksMissing = errors.Fatal("packs from index missing in repo")
-var errorSizeNotMatching = errors.Fatal("pack size does not match calculated size from index")
-
 var cmdPrune = &cobra.Command{
 	Use:   "prune [flags]",
 	Short: "Remove unneeded data from the repository",
@@ -38,7 +32,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, _ []string) error {
-		return runPrune(cmd.Context(), pruneOptions, globalOptions)
+		term, cancel := setupTermstatus()
+		defer cancel()
+		return runPrune(cmd.Context(), pruneOptions, globalOptions, term)
 	},
 }
 
@@ -138,7 +134,7 @@ func verifyPruneOptions(opts *PruneOptions) error {
 	return nil
 }
 
-func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error {
+func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, term *termstatus.Terminal) error {
 	err := verifyPruneOptions(&opts)
 	if err != nil {
 		return err
@@ -148,18 +144,11 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error
 		return errors.Fatal("disabled compression and `--repack-uncompressed` are mutually exclusive")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
 	if err != nil {
 		return err
 	}
-
-	if repo.Connections() < 2 {
-		return errors.Fatal("prune requires a backend connection limit of at least two")
-	}
-
-	if repo.Config().Version < 2 && opts.RepackUncompressed {
-		return errors.Fatal("compression requires at least repository format version 2")
-	}
+	defer unlock()
 
 	if opts.UnsafeNoSpaceRecovery != "" {
 		repoID := repo.Config().ID
@@ -169,16 +158,10 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error
 		opts.unsafeRecovery = true
 	}
 
-	lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-	defer unlockRepo(lock)
-	if err != nil {
-		return err
-	}
-
-	return runPruneWithRepo(ctx, opts, gopts, repo, restic.NewIDSet())
+	return runPruneWithRepo(ctx, opts, gopts, repo, restic.NewIDSet(), term)
 }
 
-func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet) error {
+func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet, term *termstatus.Terminal) error {
 	// we do not need index updates while pruning!
 	repo.DisableAutoIndexUpdate()
 
@@ -186,24 +169,43 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption
 		Print("warning: running prune without a cache, this may be very slow!\n")
 	}
 
-	Verbosef("loading indexes...\n")
+	printer := newTerminalProgressPrinter(gopts.verbosity, term)
+
+	printer.P("loading indexes...\n")
 	// loading the index before the snapshots is ok, as we use an exclusive lock here
-	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
 	err := repo.LoadIndex(ctx, bar)
 	if err != nil {
 		return err
 	}
 
-	plan, stats, err := planPrune(ctx, opts, repo, ignoreSnapshots, gopts.Quiet)
+	popts := repository.PruneOptions{
+		DryRun:         opts.DryRun,
+		UnsafeRecovery: opts.unsafeRecovery,
+
+		MaxUnusedBytes: opts.maxUnusedBytes,
+		MaxRepackBytes: opts.MaxRepackBytes,
+
+		RepackCachableOnly: opts.RepackCachableOnly,
+		RepackSmall:        opts.RepackSmall,
+		RepackUncompressed: opts.RepackUncompressed,
+	}
+
+	plan, err := repository.PlanPrune(ctx, popts, repo, func(ctx context.Context, repo restic.Repository) (usedBlobs restic.CountedBlobSet, err error) {
+		return getUsedBlobs(ctx, repo, ignoreSnapshots, printer)
+	}, printer)
 	if err != nil {
 		return err
 	}
-
-	if opts.DryRun {
-		Verbosef("\nWould have made the following changes:")
+	if ctx.Err() != nil {
+		return ctx.Err()
 	}
 
-	err = printPruneStats(stats)
+	if popts.DryRun {
+		printer.P("\nWould have made the following changes:")
+	}
+
+	err = printPruneStats(printer, plan.Stats())
 	if err != nil {
 		return err
 	}
@@ -211,605 +213,54 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption
 	// Trigger GC to reset garbage collection threshold
 	runtime.GC()
 
-	return doPrune(ctx, opts, gopts, repo, plan)
-}
-
-type pruneStats struct {
-	blobs struct {
-		used      uint
-		duplicate uint
-		unused    uint
-		remove    uint
-		repack    uint
-		repackrm  uint
-	}
-	size struct {
-		used         uint64
-		duplicate    uint64
-		unused       uint64
-		remove       uint64
-		repack       uint64
-		repackrm     uint64
-		unref        uint64
-		uncompressed uint64
-	}
-	packs struct {
-		used       uint
-		unused     uint
-		partlyUsed uint
-		unref      uint
-		keep       uint
-		repack     uint
-		remove     uint
-	}
-}
-
-type prunePlan struct {
-	removePacksFirst restic.IDSet          // packs to remove first (unreferenced packs)
-	repackPacks      restic.IDSet          // packs to repack
-	keepBlobs        restic.CountedBlobSet // blobs to keep during repacking
-	removePacks      restic.IDSet          // packs to remove
-	ignorePacks      restic.IDSet          // packs to ignore when rebuilding the index
-}
-
-type packInfo struct {
-	usedBlobs    uint
-	unusedBlobs  uint
-	usedSize     uint64
-	unusedSize   uint64
-	tpe          restic.BlobType
-	uncompressed bool
-}
-
-type packInfoWithID struct {
-	ID restic.ID
-	packInfo
-	mustCompress bool
-}
-
-// planPrune selects which files to rewrite and which to delete and which blobs to keep.
-// Also some summary statistics are returned.
-func planPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (prunePlan, pruneStats, error) {
-	var stats pruneStats
-
-	usedBlobs, err := getUsedBlobs(ctx, repo, ignoreSnapshots, quiet)
-	if err != nil {
-		return prunePlan{}, stats, err
-	}
-
-	Verbosef("searching used packs...\n")
-	keepBlobs, indexPack, err := packInfoFromIndex(ctx, repo.Index(), usedBlobs, &stats)
-	if err != nil {
-		return prunePlan{}, stats, err
-	}
-
-	Verbosef("collecting packs for deletion and repacking\n")
-	plan, err := decidePackAction(ctx, opts, repo, indexPack, &stats, quiet)
-	if err != nil {
-		return prunePlan{}, stats, err
-	}
-
-	if len(plan.repackPacks) != 0 {
-		blobCount := keepBlobs.Len()
-		// when repacking, we do not want to keep blobs which are
-		// already contained in kept packs, so delete them from keepBlobs
-		repo.Index().Each(ctx, func(blob restic.PackedBlob) {
-			if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) {
-				return
-			}
-			keepBlobs.Delete(blob.BlobHandle)
-		})
-
-		if keepBlobs.Len() < blobCount/2 {
-			// replace with copy to shrink map to necessary size if there's a chance to benefit
-			keepBlobs = keepBlobs.Copy()
-		}
-	} else {
-		// keepBlobs is only needed if packs are repacked
-		keepBlobs = nil
-	}
-	plan.keepBlobs = keepBlobs
-
-	return plan, stats, nil
-}
-
-func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs restic.CountedBlobSet, stats *pruneStats) (restic.CountedBlobSet, map[restic.ID]packInfo, error) {
-	// iterate over all blobs in index to find out which blobs are duplicates
-	// The counter in usedBlobs describes how many instances of the blob exist in the repository index
-	// Thus 0 == blob is missing, 1 == blob exists once, >= 2 == duplicates exist
-	idx.Each(ctx, func(blob restic.PackedBlob) {
-		bh := blob.BlobHandle
-		count, ok := usedBlobs[bh]
-		if ok {
-			if count < math.MaxUint8 {
-				// don't overflow, but saturate count at 255
-				// this can lead to a non-optimal pack selection, but won't cause
-				// problems otherwise
-				count++
-			}
-
-			usedBlobs[bh] = count
-		}
-	})
-
-	// Check if all used blobs have been found in index
-	missingBlobs := restic.NewBlobSet()
-	for bh, count := range usedBlobs {
-		if count == 0 {
-			// blob does not exist in any pack files
-			missingBlobs.Insert(bh)
-		}
-	}
-
-	if len(missingBlobs) != 0 {
-		Warnf("%v not found in the index\n\n"+
-			"Integrity check failed: Data seems to be missing.\n"+
-			"Will not start prune to prevent (additional) data loss!\n"+
-			"Please report this error (along with the output of the 'prune' run) at\n"+
-			"https://github.com/restic/restic/issues/new/choose\n", missingBlobs)
-		return nil, nil, errorIndexIncomplete
-	}
-
-	indexPack := make(map[restic.ID]packInfo)
-
-	// save computed pack header size
-	for pid, hdrSize := range pack.Size(ctx, idx, true) {
-		// initialize tpe with NumBlobTypes to indicate it's not set
-		indexPack[pid] = packInfo{tpe: restic.NumBlobTypes, usedSize: uint64(hdrSize)}
-	}
-
-	hasDuplicates := false
-	// iterate over all blobs in index to generate packInfo
-	idx.Each(ctx, func(blob restic.PackedBlob) {
-		ip := indexPack[blob.PackID]
-
-		// Set blob type if not yet set
-		if ip.tpe == restic.NumBlobTypes {
-			ip.tpe = blob.Type
-		}
-
-		// mark mixed packs with "Invalid blob type"
-		if ip.tpe != blob.Type {
-			ip.tpe = restic.InvalidBlob
-		}
-
-		bh := blob.BlobHandle
-		size := uint64(blob.Length)
-		dupCount := usedBlobs[bh]
-		switch {
-		case dupCount >= 2:
-			hasDuplicates = true
-			// mark as unused for now, we will later on select one copy
-			ip.unusedSize += size
-			ip.unusedBlobs++
-
-			// count as duplicate, will later on change one copy to be counted as used
-			stats.size.duplicate += size
-			stats.blobs.duplicate++
-		case dupCount == 1: // used blob, not duplicate
-			ip.usedSize += size
-			ip.usedBlobs++
-
-			stats.size.used += size
-			stats.blobs.used++
-		default: // unused blob
-			ip.unusedSize += size
-			ip.unusedBlobs++
-
-			stats.size.unused += size
-			stats.blobs.unused++
-		}
-		if !blob.IsCompressed() {
-			ip.uncompressed = true
-		}
-		// update indexPack
-		indexPack[blob.PackID] = ip
-	})
-
-	// if duplicate blobs exist, those will be set to either "used" or "unused":
-	// - mark only one occurrence of duplicate blobs as used
-	// - if there are already some used blobs in a pack, possibly mark duplicates in this pack as "used"
-	// - if there are no used blobs in a pack, possibly mark duplicates as "unused"
-	if hasDuplicates {
-		// iterate again over all blobs in index (this is pretty cheap, all in-mem)
-		idx.Each(ctx, func(blob restic.PackedBlob) {
-			bh := blob.BlobHandle
-			count, ok := usedBlobs[bh]
-			// skip non-duplicate, aka. normal blobs
-			// count == 0 is used to mark that this was a duplicate blob with only a single occurrence remaining
-			if !ok || count == 1 {
-				return
-			}
-
-			ip := indexPack[blob.PackID]
-			size := uint64(blob.Length)
-			switch {
-			case ip.usedBlobs > 0, count == 0:
-				// other used blobs in pack or "last" occurrence ->  transition to used
-				ip.usedSize += size
-				ip.usedBlobs++
-				ip.unusedSize -= size
-				ip.unusedBlobs--
-				// same for the global statistics
-				stats.size.used += size
-				stats.blobs.used++
-				stats.size.duplicate -= size
-				stats.blobs.duplicate--
-				// let other occurrences remain marked as unused
-				usedBlobs[bh] = 1
-			default:
-				// remain unused and decrease counter
-				count--
-				if count == 1 {
-					// setting count to 1 would lead to forgetting that this blob had duplicates
-					// thus use the special value zero. This will select the last instance of the blob for keeping.
-					count = 0
-				}
-				usedBlobs[bh] = count
-			}
-			// update indexPack
-			indexPack[blob.PackID] = ip
-		})
-	}
-
-	// Sanity check. If no duplicates exist, all blobs have value 1. After handling
-	// duplicates, this also applies to duplicates.
-	for _, count := range usedBlobs {
-		if count != 1 {
-			panic("internal error during blob selection")
-		}
-	}
-
-	return usedBlobs, indexPack, nil
-}
-
-func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Repository, indexPack map[restic.ID]packInfo, stats *pruneStats, quiet bool) (prunePlan, error) {
-	removePacksFirst := restic.NewIDSet()
-	removePacks := restic.NewIDSet()
-	repackPacks := restic.NewIDSet()
-
-	var repackCandidates []packInfoWithID
-	var repackSmallCandidates []packInfoWithID
-	repoVersion := repo.Config().Version
-	// only repack very small files by default
-	targetPackSize := repo.PackSize() / 25
-	if opts.RepackSmall {
-		// consider files with at least 80% of the target size as large enough
-		targetPackSize = repo.PackSize() / 5 * 4
-	}
-
-	// loop over all packs and decide what to do
-	bar := newProgressMax(!quiet, uint64(len(indexPack)), "packs processed")
-	err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
-		p, ok := indexPack[id]
-		if !ok {
-			// Pack was not referenced in index and is not used  => immediately remove!
-			Verboseff("will remove pack %v as it is unused and not indexed\n", id.Str())
-			removePacksFirst.Insert(id)
-			stats.size.unref += uint64(packSize)
-			return nil
-		}
-
-		if p.unusedSize+p.usedSize != uint64(packSize) && p.usedBlobs != 0 {
-			// Pack size does not fit and pack is needed => error
-			// If the pack is not needed, this is no error, the pack can
-			// and will be simply removed, see below.
-			Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic repair index'.\n",
-				id.Str(), p.unusedSize+p.usedSize, packSize)
-			return errorSizeNotMatching
-		}
-
-		// statistics
-		switch {
-		case p.usedBlobs == 0:
-			stats.packs.unused++
-		case p.unusedBlobs == 0:
-			stats.packs.used++
-		default:
-			stats.packs.partlyUsed++
-		}
-
-		if p.uncompressed {
-			stats.size.uncompressed += p.unusedSize + p.usedSize
-		}
-		mustCompress := false
-		if repoVersion >= 2 {
-			// repo v2: always repack tree blobs if uncompressed
-			// compress data blobs if requested
-			mustCompress = (p.tpe == restic.TreeBlob || opts.RepackUncompressed) && p.uncompressed
-		}
-
-		// decide what to do
-		switch {
-		case p.usedBlobs == 0:
-			// All blobs in pack are no longer used => remove pack!
-			removePacks.Insert(id)
-			stats.blobs.remove += p.unusedBlobs
-			stats.size.remove += p.unusedSize
-
-		case opts.RepackCachableOnly && p.tpe == restic.DataBlob:
-			// if this is a data pack and --repack-cacheable-only is set => keep pack!
-			stats.packs.keep++
-
-		case p.unusedBlobs == 0 && p.tpe != restic.InvalidBlob && !mustCompress:
-			if packSize >= int64(targetPackSize) {
-				// All blobs in pack are used and not mixed => keep pack!
-				stats.packs.keep++
-			} else {
-				repackSmallCandidates = append(repackSmallCandidates, packInfoWithID{ID: id, packInfo: p, mustCompress: mustCompress})
-			}
-
-		default:
-			// all other packs are candidates for repacking
-			repackCandidates = append(repackCandidates, packInfoWithID{ID: id, packInfo: p, mustCompress: mustCompress})
-		}
-
-		delete(indexPack, id)
-		bar.Add(1)
-		return nil
-	})
-	bar.Done()
-	if err != nil {
-		return prunePlan{}, err
-	}
-
-	// At this point indexPacks contains only missing packs!
-
-	// missing packs that are not needed can be ignored
-	ignorePacks := restic.NewIDSet()
-	for id, p := range indexPack {
-		if p.usedBlobs == 0 {
-			ignorePacks.Insert(id)
-			stats.blobs.remove += p.unusedBlobs
-			stats.size.remove += p.unusedSize
-			delete(indexPack, id)
-		}
-	}
-
-	if len(indexPack) != 0 {
-		Warnf("The index references %d needed pack files which are missing from the repository:\n", len(indexPack))
-		for id := range indexPack {
-			Warnf("  %v\n", id)
-		}
-		return prunePlan{}, errorPacksMissing
-	}
-	if len(ignorePacks) != 0 {
-		Warnf("Missing but unneeded pack files are referenced in the index, will be repaired\n")
-		for id := range ignorePacks {
-			Warnf("will forget missing pack file %v\n", id)
-		}
-	}
-
-	if len(repackSmallCandidates) < 10 {
-		// too few small files to be worth the trouble, this also prevents endlessly repacking
-		// if there is just a single pack file below the target size
-		stats.packs.keep += uint(len(repackSmallCandidates))
-	} else {
-		repackCandidates = append(repackCandidates, repackSmallCandidates...)
-	}
-
-	// Sort repackCandidates such that packs with highest ratio unused/used space are picked first.
-	// This is equivalent to sorting by unused / total space.
-	// Instead of unused[i] / used[i] > unused[j] / used[j] we use
-	// unused[i] * used[j] > unused[j] * used[i] as uint32*uint32 < uint64
-	// Moreover packs containing trees and too small packs are sorted to the beginning
-	sort.Slice(repackCandidates, func(i, j int) bool {
-		pi := repackCandidates[i].packInfo
-		pj := repackCandidates[j].packInfo
-		switch {
-		case pi.tpe != restic.DataBlob && pj.tpe == restic.DataBlob:
-			return true
-		case pj.tpe != restic.DataBlob && pi.tpe == restic.DataBlob:
-			return false
-		case pi.unusedSize+pi.usedSize < uint64(targetPackSize) && pj.unusedSize+pj.usedSize >= uint64(targetPackSize):
-			return true
-		case pj.unusedSize+pj.usedSize < uint64(targetPackSize) && pi.unusedSize+pi.usedSize >= uint64(targetPackSize):
-			return false
-		}
-		return pi.unusedSize*pj.usedSize > pj.unusedSize*pi.usedSize
-	})
-
-	repack := func(id restic.ID, p packInfo) {
-		repackPacks.Insert(id)
-		stats.blobs.repack += p.unusedBlobs + p.usedBlobs
-		stats.size.repack += p.unusedSize + p.usedSize
-		stats.blobs.repackrm += p.unusedBlobs
-		stats.size.repackrm += p.unusedSize
-		if p.uncompressed {
-			stats.size.uncompressed -= p.unusedSize + p.usedSize
-		}
-	}
-
-	// calculate limit for number of unused bytes in the repo after repacking
-	maxUnusedSizeAfter := opts.maxUnusedBytes(stats.size.used)
-
-	for _, p := range repackCandidates {
-		reachedUnusedSizeAfter := (stats.size.unused-stats.size.remove-stats.size.repackrm < maxUnusedSizeAfter)
-		reachedRepackSize := stats.size.repack+p.unusedSize+p.usedSize >= opts.MaxRepackBytes
-		packIsLargeEnough := p.unusedSize+p.usedSize >= uint64(targetPackSize)
-
-		switch {
-		case reachedRepackSize:
-			stats.packs.keep++
-
-		case p.tpe != restic.DataBlob, p.mustCompress:
-			// repacking non-data packs / uncompressed-trees is only limited by repackSize
-			repack(p.ID, p.packInfo)
-
-		case reachedUnusedSizeAfter && packIsLargeEnough:
-			// for all other packs stop repacking if tolerated unused size is reached.
-			stats.packs.keep++
-
-		default:
-			repack(p.ID, p.packInfo)
-		}
-	}
-
-	stats.packs.unref = uint(len(removePacksFirst))
-	stats.packs.repack = uint(len(repackPacks))
-	stats.packs.remove = uint(len(removePacks))
-
-	if repo.Config().Version < 2 {
-		// compression not supported for repository format version 1
-		stats.size.uncompressed = 0
-	}
-
-	return prunePlan{removePacksFirst: removePacksFirst,
-		removePacks: removePacks,
-		repackPacks: repackPacks,
-		ignorePacks: ignorePacks,
-	}, nil
+	return plan.Execute(ctx, printer)
 }
 
 // printPruneStats prints out the statistics
-func printPruneStats(stats pruneStats) error {
-	Verboseff("\nused:         %10d blobs / %s\n", stats.blobs.used, ui.FormatBytes(stats.size.used))
-	if stats.blobs.duplicate > 0 {
-		Verboseff("duplicates:   %10d blobs / %s\n", stats.blobs.duplicate, ui.FormatBytes(stats.size.duplicate))
+func printPruneStats(printer progress.Printer, stats repository.PruneStats) error {
+	printer.V("\nused:         %10d blobs / %s\n", stats.Blobs.Used, ui.FormatBytes(stats.Size.Used))
+	if stats.Blobs.Duplicate > 0 {
+		printer.V("duplicates:   %10d blobs / %s\n", stats.Blobs.Duplicate, ui.FormatBytes(stats.Size.Duplicate))
 	}
-	Verboseff("unused:       %10d blobs / %s\n", stats.blobs.unused, ui.FormatBytes(stats.size.unused))
-	if stats.size.unref > 0 {
-		Verboseff("unreferenced:                    %s\n", ui.FormatBytes(stats.size.unref))
+	printer.V("unused:       %10d blobs / %s\n", stats.Blobs.Unused, ui.FormatBytes(stats.Size.Unused))
+	if stats.Size.Unref > 0 {
+		printer.V("unreferenced:                    %s\n", ui.FormatBytes(stats.Size.Unref))
 	}
-	totalBlobs := stats.blobs.used + stats.blobs.unused + stats.blobs.duplicate
-	totalSize := stats.size.used + stats.size.duplicate + stats.size.unused + stats.size.unref
-	unusedSize := stats.size.duplicate + stats.size.unused
-	Verboseff("total:        %10d blobs / %s\n", totalBlobs, ui.FormatBytes(totalSize))
-	Verboseff("unused size: %s of total size\n", ui.FormatPercent(unusedSize, totalSize))
+	totalBlobs := stats.Blobs.Used + stats.Blobs.Unused + stats.Blobs.Duplicate
+	totalSize := stats.Size.Used + stats.Size.Duplicate + stats.Size.Unused + stats.Size.Unref
+	unusedSize := stats.Size.Duplicate + stats.Size.Unused
+	printer.V("total:        %10d blobs / %s\n", totalBlobs, ui.FormatBytes(totalSize))
+	printer.V("unused size: %s of total size\n", ui.FormatPercent(unusedSize, totalSize))
 
-	Verbosef("\nto repack:    %10d blobs / %s\n", stats.blobs.repack, ui.FormatBytes(stats.size.repack))
-	Verbosef("this removes: %10d blobs / %s\n", stats.blobs.repackrm, ui.FormatBytes(stats.size.repackrm))
-	Verbosef("to delete:    %10d blobs / %s\n", stats.blobs.remove, ui.FormatBytes(stats.size.remove+stats.size.unref))
-	totalPruneSize := stats.size.remove + stats.size.repackrm + stats.size.unref
-	Verbosef("total prune:  %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, ui.FormatBytes(totalPruneSize))
-	if stats.size.uncompressed > 0 {
-		Verbosef("not yet compressed:              %s\n", ui.FormatBytes(stats.size.uncompressed))
+	printer.P("\nto repack:    %10d blobs / %s\n", stats.Blobs.Repack, ui.FormatBytes(stats.Size.Repack))
+	printer.P("this removes: %10d blobs / %s\n", stats.Blobs.Repackrm, ui.FormatBytes(stats.Size.Repackrm))
+	printer.P("to delete:    %10d blobs / %s\n", stats.Blobs.Remove, ui.FormatBytes(stats.Size.Remove+stats.Size.Unref))
+	totalPruneSize := stats.Size.Remove + stats.Size.Repackrm + stats.Size.Unref
+	printer.P("total prune:  %10d blobs / %s\n", stats.Blobs.Remove+stats.Blobs.Repackrm, ui.FormatBytes(totalPruneSize))
+	if stats.Size.Uncompressed > 0 {
+		printer.P("not yet compressed:              %s\n", ui.FormatBytes(stats.Size.Uncompressed))
 	}
-	Verbosef("remaining:    %10d blobs / %s\n", totalBlobs-(stats.blobs.remove+stats.blobs.repackrm), ui.FormatBytes(totalSize-totalPruneSize))
-	unusedAfter := unusedSize - stats.size.remove - stats.size.repackrm
-	Verbosef("unused size after prune: %s (%s of remaining size)\n",
+	printer.P("remaining:    %10d blobs / %s\n", totalBlobs-(stats.Blobs.Remove+stats.Blobs.Repackrm), ui.FormatBytes(totalSize-totalPruneSize))
+	unusedAfter := unusedSize - stats.Size.Remove - stats.Size.Repackrm
+	printer.P("unused size after prune: %s (%s of remaining size)\n",
 		ui.FormatBytes(unusedAfter), ui.FormatPercent(unusedAfter, totalSize-totalPruneSize))
-	Verbosef("\n")
-	Verboseff("totally used packs: %10d\n", stats.packs.used)
-	Verboseff("partly used packs:  %10d\n", stats.packs.partlyUsed)
-	Verboseff("unused packs:       %10d\n\n", stats.packs.unused)
+	printer.P("\n")
+	printer.V("totally used packs: %10d\n", stats.Packs.Used)
+	printer.V("partly used packs:  %10d\n", stats.Packs.PartlyUsed)
+	printer.V("unused packs:       %10d\n\n", stats.Packs.Unused)
 
-	Verboseff("to keep:      %10d packs\n", stats.packs.keep)
-	Verboseff("to repack:    %10d packs\n", stats.packs.repack)
-	Verboseff("to delete:    %10d packs\n", stats.packs.remove)
-	if stats.packs.unref > 0 {
-		Verboseff("to delete:    %10d unreferenced packs\n\n", stats.packs.unref)
+	printer.V("to keep:      %10d packs\n", stats.Packs.Keep)
+	printer.V("to repack:    %10d packs\n", stats.Packs.Repack)
+	printer.V("to delete:    %10d packs\n", stats.Packs.Remove)
+	if stats.Packs.Unref > 0 {
+		printer.V("to delete:    %10d unreferenced packs\n\n", stats.Packs.Unref)
 	}
 	return nil
 }
 
-// doPrune does the actual pruning:
-// - remove unreferenced packs first
-// - repack given pack files while keeping the given blobs
-// - rebuild the index while ignoring all files that will be deleted
-// - delete the files
-// plan.removePacks and plan.ignorePacks are modified in this function.
-func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo restic.Repository, plan prunePlan) (err error) {
-	if opts.DryRun {
-		if !gopts.JSON && gopts.verbosity >= 2 {
-			Printf("Repeated prune dry-runs can report slightly different amounts of data to keep or repack. This is expected behavior.\n\n")
-			if len(plan.removePacksFirst) > 0 {
-				Printf("Would have removed the following unreferenced packs:\n%v\n\n", plan.removePacksFirst)
-			}
-			Printf("Would have repacked and removed the following packs:\n%v\n\n", plan.repackPacks)
-			Printf("Would have removed the following no longer used packs:\n%v\n\n", plan.removePacks)
-		}
-		// Always quit here if DryRun was set!
-		return nil
-	}
-
-	// unreferenced packs can be safely deleted first
-	if len(plan.removePacksFirst) != 0 {
-		Verbosef("deleting unreferenced packs\n")
-		DeleteFiles(ctx, gopts, repo, plan.removePacksFirst, restic.PackFile)
-	}
-
-	if len(plan.repackPacks) != 0 {
-		Verbosef("repacking packs\n")
-		bar := newProgressMax(!gopts.Quiet, uint64(len(plan.repackPacks)), "packs repacked")
-		_, err := repository.Repack(ctx, repo, repo, plan.repackPacks, plan.keepBlobs, bar)
-		bar.Done()
-		if err != nil {
-			return errors.Fatal(err.Error())
-		}
-
-		// Also remove repacked packs
-		plan.removePacks.Merge(plan.repackPacks)
-
-		if len(plan.keepBlobs) != 0 {
-			Warnf("%v was not repacked\n\n"+
-				"Integrity check failed.\n"+
-				"Please report this error (along with the output of the 'prune' run) at\n"+
-				"https://github.com/restic/restic/issues/new/choose\n", plan.keepBlobs)
-			return errors.Fatal("internal error: blobs were not repacked")
-		}
-
-		// allow GC of the blob set
-		plan.keepBlobs = nil
-	}
-
-	if len(plan.ignorePacks) == 0 {
-		plan.ignorePacks = plan.removePacks
-	} else {
-		plan.ignorePacks.Merge(plan.removePacks)
-	}
-
-	if opts.unsafeRecovery {
-		Verbosef("deleting index files\n")
-		indexFiles := repo.Index().(*index.MasterIndex).IDs()
-		err = DeleteFilesChecked(ctx, gopts, repo, indexFiles, restic.IndexFile)
-		if err != nil {
-			return errors.Fatalf("%s", err)
-		}
-	} else if len(plan.ignorePacks) != 0 {
-		err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil, false)
-		if err != nil {
-			return errors.Fatalf("%s", err)
-		}
-	}
-
-	if len(plan.removePacks) != 0 {
-		Verbosef("removing %d old packs\n", len(plan.removePacks))
-		DeleteFiles(ctx, gopts, repo, plan.removePacks, restic.PackFile)
-	}
-
-	if opts.unsafeRecovery {
-		err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil, true)
-		if err != nil {
-			return errors.Fatalf("%s", err)
-		}
-	}
-
-	Verbosef("done\n")
-	return nil
-}
-
-func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs, skipDeletion bool) error {
-	Verbosef("rebuilding index\n")
-
-	bar := newProgressMax(!gopts.Quiet, 0, "packs processed")
-	return repo.Index().Save(ctx, repo, removePacks, extraObsolete, restic.MasterIndexSaveOpts{
-		SaveProgress: bar,
-		DeleteProgress: func() *progress.Counter {
-			return newProgressMax(!gopts.Quiet, 0, "old indexes deleted")
-		},
-		DeleteReport: func(id restic.ID, _ error) {
-			if gopts.verbosity > 2 {
-				Verbosef("removed index %v\n", id.String())
-			}
-		},
-		SkipDeletion: skipDeletion,
-	})
-}
-
-func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (usedBlobs restic.CountedBlobSet, err error) {
+func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, printer progress.Printer) (usedBlobs restic.CountedBlobSet, err error) {
 	var snapshotTrees restic.IDs
-	Verbosef("loading all snapshots...\n")
+	printer.P("loading all snapshots...\n")
 	err = restic.ForAllSnapshots(ctx, repo, repo, ignoreSnapshots,
 		func(id restic.ID, sn *restic.Snapshot, err error) error {
 			if err != nil {
@@ -824,11 +275,12 @@ func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots r
 		return nil, errors.Fatalf("failed loading snapshot: %v", err)
 	}
 
-	Verbosef("finding data that is still in use for %d snapshots\n", len(snapshotTrees))
+	printer.P("finding data that is still in use for %d snapshots\n", len(snapshotTrees))
 
 	usedBlobs = restic.NewCountedBlobSet()
 
-	bar := newProgressMax(!quiet, uint64(len(snapshotTrees)), "snapshots")
+	bar := printer.NewCounter("snapshots")
+	bar.SetMax(uint64(len(snapshotTrees)))
 	defer bar.Done()
 
 	err = restic.FindUsedBlobs(ctx, repo, snapshotTrees, usedBlobs, bar)

cmd/restic/cmd_prune_integration_test.go

@@ -7,7 +7,9 @@ import (
 	"testing"
 
 	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/repository"
 	rtest "github.com/restic/restic/internal/test"
+	"github.com/restic/restic/internal/ui/termstatus"
 )
 
 func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
@@ -16,7 +18,9 @@ func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
 	defer func() {
 		gopts.backendTestHook = oldHook
 	}()
-	rtest.OK(t, runPrune(context.TODO(), opts, gopts))
+	rtest.OK(t, withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+		return runPrune(context.TODO(), opts, gopts, term)
+	}))
 }
 
 func TestPrune(t *testing.T) {
@@ -31,7 +35,7 @@ func testPruneVariants(t *testing.T, unsafeNoSpaceRecovery bool) {
 	}
 	t.Run("0"+suffix, func(t *testing.T) {
 		opts := PruneOptions{MaxUnused: "0%", unsafeRecovery: unsafeNoSpaceRecovery}
-		checkOpts := CheckOptions{ReadData: true, CheckUnused: true}
+		checkOpts := CheckOptions{ReadData: true, CheckUnused: !unsafeNoSpaceRecovery}
 		testPrune(t, opts, checkOpts)
 	})
 
@@ -84,7 +88,9 @@ func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) {
 		pruneOpts := PruneOptions{
 			MaxUnused: "5%",
 		}
-		return runForget(context.TODO(), opts, pruneOpts, gopts, args)
+		return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+			return runForget(context.TODO(), opts, pruneOpts, gopts, term, args)
+		})
 	})
 	rtest.OK(t, err)
 
@@ -138,7 +144,9 @@ func TestPruneWithDamagedRepository(t *testing.T) {
 		env.gopts.backendTestHook = oldHook
 	}()
 	// prune should fail
-	rtest.Assert(t, runPrune(context.TODO(), pruneDefaultOptions, env.gopts) == errorPacksMissing,
+	rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+		return runPrune(context.TODO(), pruneDefaultOptions, env.gopts, term)
+	}) == repository.ErrPacksMissing,
 		"prune should have reported index not complete error")
 }
 
@@ -218,7 +226,9 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
 		testRunPrune(t, env.gopts, optionsPrune)
 		testRunCheck(t, env.gopts)
 	} else {
-		rtest.Assert(t, runPrune(context.TODO(), optionsPrune, env.gopts) != nil,
+		rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+			return runPrune(context.TODO(), optionsPrune, env.gopts, term)
+		}) != nil,
 			"prune should have reported an error")
 	}
 }

cmd/restic/cmd_recover.go

@@ -40,16 +40,11 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 		return err
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
-	if err != nil {
-		return err
-	}
-
-	lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-	defer unlockRepo(lock)
+	ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, false)
 	if err != nil {
 		return err
 	}
+	defer unlock()
 
 	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
@@ -66,16 +61,22 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 	// tree. If it is not referenced, we have a root tree.
 	trees := make(map[restic.ID]bool)
 
-	repo.Index().Each(ctx, func(blob restic.PackedBlob) {
+	err = repo.Index().Each(ctx, func(blob restic.PackedBlob) {
 		if blob.Type == restic.TreeBlob {
 			trees[blob.Blob.ID] = false
 		}
 	})
+	if err != nil {
+		return err
+	}
 
 	Verbosef("load %d trees\n", len(trees))
 	bar = newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
 	for id := range trees {
 		tree, err := restic.LoadTree(ctx, repo, id)
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
 		if err != nil {
 			Warnf("unable to load tree %v: %v\n", id.Str(), err)
 			continue

cmd/restic/cmd_repair_index.go

@@ -3,10 +3,8 @@ package main
 import (
 	"context"
 
-	"github.com/restic/restic/internal/index"
-	"github.com/restic/restic/internal/pack"
 	"github.com/restic/restic/internal/repository"
-	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui/termstatus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 )
@@ -25,7 +23,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, _ []string) error {
-		return runRebuildIndex(cmd.Context(), repairIndexOptions, globalOptions)
+		term, cancel := setupTermstatus()
+		defer cancel()
+		return runRebuildIndex(cmd.Context(), repairIndexOptions, globalOptions, term)
 	},
 }
 
@@ -55,110 +55,22 @@ func init() {
 	}
 }
 
-func runRebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions) error {
-	repo, err := OpenRepository(ctx, gopts)
+func runRebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions, term *termstatus.Terminal) error {
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
+	if err != nil {
+		return err
+	}
+	defer unlock()
+
+	printer := newTerminalProgressPrinter(gopts.verbosity, term)
+
+	err = repository.RepairIndex(ctx, repo, repository.RepairIndexOptions{
+		ReadAllPacks: opts.ReadAllPacks,
+	}, printer)
 	if err != nil {
 		return err
 	}
 
-	lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-	defer unlockRepo(lock)
-	if err != nil {
-		return err
-	}
-
-	return rebuildIndex(ctx, opts, gopts, repo)
-}
-
-func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions, repo *repository.Repository) error {
-	var obsoleteIndexes restic.IDs
-	packSizeFromList := make(map[restic.ID]int64)
-	packSizeFromIndex := make(map[restic.ID]int64)
-	removePacks := restic.NewIDSet()
-
-	if opts.ReadAllPacks {
-		// get list of old index files but start with empty index
-		err := repo.List(ctx, restic.IndexFile, func(id restic.ID, _ int64) error {
-			obsoleteIndexes = append(obsoleteIndexes, id)
-			return nil
-		})
-		if err != nil {
-			return err
-		}
-	} else {
-		Verbosef("loading indexes...\n")
-		mi := index.NewMasterIndex()
-		err := index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, _ bool, err error) error {
-			if err != nil {
-				Warnf("removing invalid index %v: %v\n", id, err)
-				obsoleteIndexes = append(obsoleteIndexes, id)
-				return nil
-			}
-
-			mi.Insert(idx)
-			return nil
-		})
-		if err != nil {
-			return err
-		}
-
-		err = mi.MergeFinalIndexes()
-		if err != nil {
-			return err
-		}
-
-		err = repo.SetIndex(mi)
-		if err != nil {
-			return err
-		}
-		packSizeFromIndex = pack.Size(ctx, repo.Index(), false)
-	}
-
-	Verbosef("getting pack files to read...\n")
-	err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
-		size, ok := packSizeFromIndex[id]
-		if !ok || size != packSize {
-			// Pack was not referenced in index or size does not match
-			packSizeFromList[id] = packSize
-			removePacks.Insert(id)
-		}
-		if !ok {
-			Warnf("adding pack file to index %v\n", id)
-		} else if size != packSize {
-			Warnf("reindexing pack file %v with unexpected size %v instead of %v\n", id, packSize, size)
-		}
-		delete(packSizeFromIndex, id)
-		return nil
-	})
-	if err != nil {
-		return err
-	}
-	for id := range packSizeFromIndex {
-		// forget pack files that are referenced in the index but do not exist
-		// when rebuilding the index
-		removePacks.Insert(id)
-		Warnf("removing not found pack file %v\n", id)
-	}
-
-	if len(packSizeFromList) > 0 {
-		Verbosef("reading pack files\n")
-		bar := newProgressMax(!gopts.Quiet, uint64(len(packSizeFromList)), "packs")
-		invalidFiles, err := repo.CreateIndexFromPacks(ctx, packSizeFromList, bar)
-		bar.Done()
-		if err != nil {
-			return err
-		}
-
-		for _, id := range invalidFiles {
-			Verboseff("skipped incomplete pack file: %v\n", id)
-		}
-	}
-
-	err = rebuildIndexFiles(ctx, gopts, repo, removePacks, obsoleteIndexes, false)
-	if err != nil {
-		return err
-	}
-	Verbosef("done\n")
-
+	printer.P("done\n")
 	return nil
 }

cmd/restic/cmd_repair_index_integration_test.go

@@ -13,12 +13,15 @@ import (
 	"github.com/restic/restic/internal/index"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
+	"github.com/restic/restic/internal/ui/termstatus"
 )
 
 func testRunRebuildIndex(t testing.TB, gopts GlobalOptions) {
 	rtest.OK(t, withRestoreGlobalOptions(func() error {
-		globalOptions.stdout = io.Discard
-		return runRebuildIndex(context.TODO(), RepairIndexOptions{}, gopts)
+		return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+			globalOptions.stdout = io.Discard
+			return runRebuildIndex(context.TODO(), RepairIndexOptions{}, gopts, term)
+		})
 	}))
 }
 
@@ -126,12 +129,13 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) {
 	rtest.SetupTarTestFixture(t, env.base, datafile)
 
 	err := withRestoreGlobalOptions(func() error {
-		globalOptions.stdout = io.Discard
-
 		env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
 			return &appendOnlyBackend{r}, nil
 		}
-		return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts)
+		return withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+			globalOptions.stdout = io.Discard
+			return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts, term)
+		})
 	})
 
 	if err == nil {

cmd/restic/cmd_repair_packs.go

@@ -52,25 +52,20 @@ func runRepairPacks(ctx context.Context, gopts GlobalOptions, term *termstatus.T
 		return errors.Fatal("no ids specified")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
 	if err != nil {
 		return err
 	}
+	defer unlock()
 
-	lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-	defer unlockRepo(lock)
-	if err != nil {
-		return err
-	}
+	printer := newTerminalProgressPrinter(gopts.verbosity, term)
 
-	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
 	err = repo.LoadIndex(ctx, bar)
 	if err != nil {
 		return errors.Fatalf("%s", err)
 	}
 
-	printer := newTerminalProgressPrinter(gopts.verbosity, term)
-
 	printer.P("saving backup copies of pack files to current folder")
 	for id := range ids {
 		f, err := os.OpenFile("pack-"+id.String(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o666)
@@ -87,6 +82,10 @@ func runRepairPacks(ctx context.Context, gopts GlobalOptions, term *termstatus.T
 			return err
 		})
 		if err != nil {
+			_ = f.Close()
+			return err
+		}
+		if err := f.Close(); err != nil {
 			return err
 		}
 	}

cmd/restic/cmd_repair_snapshots.go

@@ -66,22 +66,11 @@ func init() {
 }
 
 func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOptions, args []string) error {
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun)
 	if err != nil {
 		return err
 	}
-
-	if !opts.DryRun {
-		var lock *restic.Lock
-		var err error
-		lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	} else {
-		repo.SetDryRun()
-	}
+	defer unlock()
 
 	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
@@ -156,6 +145,9 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
 			changedCount++
 		}
 	}
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 
 	Verbosef("\n")
 	if changedCount == 0 {

cmd/restic/cmd_restore.go

@@ -127,19 +127,11 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
 
 	debug.Log("restore %v to %v", snapshotIDString, opts.Target)
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
-
-	if !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
+	defer unlock()
 
 	sn, subfolder, err := (&restic.SnapshotFilter{
 		Hosts: opts.Hosts,

cmd/restic/cmd_restore_integration_test.go

@@ -4,13 +4,14 @@ import (
 	"context"
 	"fmt"
 	"io"
-	mrand "math/rand"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"syscall"
 	"testing"
 	"time"
 
+	"github.com/restic/restic/internal/feature"
 	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
@@ -116,7 +117,7 @@ func TestRestore(t *testing.T) {
 	for i := 0; i < 10; i++ {
 		p := filepath.Join(env.testdata, fmt.Sprintf("foo/bar/testfile%v", i))
 		rtest.OK(t, os.MkdirAll(filepath.Dir(p), 0755))
-		rtest.OK(t, appendRandomData(p, uint(mrand.Intn(2<<21))))
+		rtest.OK(t, appendRandomData(p, uint(rand.Intn(2<<21))))
 	}
 
 	opts := BackupOptions{}
@@ -274,6 +275,7 @@ func TestRestoreNoMetadataOnIgnoredIntermediateDirs(t *testing.T) {
 }
 
 func TestRestoreLocalLayout(t *testing.T) {
+	defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)()
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
 

cmd/restic/cmd_rewrite.go

@@ -256,27 +256,22 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a
 		return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	var (
+		repo   *repository.Repository
+		unlock func()
+		err    error
+	)
+
+	if opts.Forget {
+		Verbosef("create exclusive lock for repository\n")
+		ctx, repo, unlock, err = openWithExclusiveLock(ctx, gopts, opts.DryRun)
+	} else {
+		ctx, repo, unlock, err = openWithAppendLock(ctx, gopts, opts.DryRun)
+	}
 	if err != nil {
 		return err
 	}
-
-	if !opts.DryRun {
-		var lock *restic.Lock
-		var err error
-		if opts.Forget {
-			Verbosef("create exclusive lock for repository\n")
-			lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-		} else {
-			lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		}
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	} else {
-		repo.SetDryRun()
-	}
+	defer unlock()
 
 	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
@@ -299,6 +294,9 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a
 			changedCount++
 		}
 	}
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 
 	Verbosef("\n")
 	if changedCount == 0 {

cmd/restic/cmd_rewrite_integration_test.go

@@ -78,8 +78,11 @@ func testRewriteMetadata(t *testing.T, metadata snapshotMetadataArgs) {
 	createBasicRewriteRepo(t, env)
 	testRunRewriteExclude(t, env.gopts, []string{}, true, metadata)
 
-	repo, _ := OpenRepository(context.TODO(), env.gopts)
-	snapshots, err := restic.TestLoadAllSnapshots(context.TODO(), repo, nil)
+	ctx, repo, unlock, err := openWithReadLock(context.TODO(), env.gopts, false)
+	rtest.OK(t, err)
+	defer unlock()
+
+	snapshots, err := restic.TestLoadAllSnapshots(ctx, repo, nil)
 	rtest.OK(t, err)
 	rtest.Assert(t, len(snapshots) == 1, "expected one snapshot, got %v", len(snapshots))
 	newSnapshot := snapshots[0]

cmd/restic/cmd_snapshots.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui"
 	"github.com/restic/restic/internal/ui/table"
 	"github.com/spf13/cobra"
 )
@@ -58,24 +59,19 @@ func init() {
 }
 
 func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions, args []string) error {
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
-
-	if !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
+	defer unlock()
 
 	var snapshots restic.Snapshots
 	for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
 		snapshots = append(snapshots, sn)
 	}
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 	snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy)
 	if err != nil {
 		return err
@@ -163,6 +159,11 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
 			keepReasons[*id] = reasons[i]
 		}
 	}
+	// check if any snapshot contains a summary
+	hasSize := false
+	for _, sn := range list {
+		hasSize = hasSize || (sn.Summary != nil)
+	}
 
 	// always sort the snapshots so that the newer ones are listed last
 	sort.SliceStable(list, func(i, j int) bool {
@@ -198,6 +199,9 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
 			tab.AddColumn("Reasons", `{{ join .Reasons "\n" }}`)
 		}
 		tab.AddColumn("Paths", `{{ join .Paths "\n" }}`)
+		if hasSize {
+			tab.AddColumn("Size", `{{ .Size }}`)
+		}
 	}
 
 	type snapshot struct {
@@ -207,6 +211,7 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
 		Tags      []string
 		Reasons   []string
 		Paths     []string
+		Size      string
 	}
 
 	var multiline bool
@@ -228,6 +233,10 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
 			multiline = true
 		}
 
+		if sn.Summary != nil {
+			data.Size = ui.FormatBytes(sn.Summary.TotalBytesProcessed)
+		}
+
 		tab.AddRow(data)
 	}
 

cmd/restic/cmd_stats.go

@@ -38,7 +38,7 @@ depending on what you are trying to calculate.
 The modes are:
 
 * restore-size: (default) Counts the size of the restored files.
-* files-by-contents: Counts total size of files, where a file is
+* files-by-contents: Counts total size of unique files, where a file is
    considered unique if it has unique contents.
 * raw-data: Counts the size of blobs in the repository, regardless of
   how many files reference them.
@@ -80,19 +80,11 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args
 		return err
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
-
-	if !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
+	defer unlock()
 
 	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
@@ -125,9 +117,8 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args
 			return fmt.Errorf("error walking snapshot: %v", err)
 		}
 	}
-
-	if err != nil {
-		return err
+	if ctx.Err() != nil {
+		return ctx.Err()
 	}
 
 	if opts.countMode == countModeRawData {
@@ -270,11 +261,14 @@ func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer,
 			// will still be restored
 			stats.TotalFileCount++
 
-			// if inodes are present, only count each inode once
-			// (hard links do not increase restore size)
-			if !hardLinkIndex.Has(node.Inode, node.DeviceID) || node.Inode == 0 {
-				hardLinkIndex.Add(node.Inode, node.DeviceID, struct{}{})
+			if node.Links == 1 || node.Type == "dir" {
 				stats.TotalSize += node.Size
+			} else {
+				// if hardlinks are present only count each deviceID+inode once
+				if !hardLinkIndex.Has(node.Inode, node.DeviceID) || node.Inode == 0 {
+					hardLinkIndex.Add(node.Inode, node.DeviceID, struct{}{})
+					stats.TotalSize += node.Size
+				}
 			}
 		}
 
@@ -357,7 +351,10 @@ func statsDebug(ctx context.Context, repo restic.Repository) error {
 		Warnf("File Type: %v\n%v\n", t, hist)
 	}
 
-	hist := statsDebugBlobs(ctx, repo)
+	hist, err := statsDebugBlobs(ctx, repo)
+	if err != nil {
+		return err
+	}
 	for _, t := range []restic.BlobType{restic.DataBlob, restic.TreeBlob} {
 		Warnf("Blob Type: %v\n%v\n\n", t, hist[t])
 	}
@@ -375,17 +372,17 @@ func statsDebugFileType(ctx context.Context, repo restic.Lister, tpe restic.File
 	return hist, err
 }
 
-func statsDebugBlobs(ctx context.Context, repo restic.Repository) [restic.NumBlobTypes]*sizeHistogram {
+func statsDebugBlobs(ctx context.Context, repo restic.Repository) ([restic.NumBlobTypes]*sizeHistogram, error) {
 	var hist [restic.NumBlobTypes]*sizeHistogram
 	for i := 0; i < len(hist); i++ {
 		hist[i] = newSizeHistogram(2 * chunker.MaxSize)
 	}
 
-	repo.Index().Each(ctx, func(pb restic.PackedBlob) {
+	err := repo.Index().Each(ctx, func(pb restic.PackedBlob) {
 		hist[pb.Type].Add(uint64(pb.Length))
 	})
 
-	return hist
+	return hist, err
 }
 
 type sizeClass struct {

cmd/restic/cmd_tag.go

@@ -104,20 +104,12 @@ func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []st
 		return errors.Fatal("--set and --add/--remove cannot be given at the same time")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	Verbosef("create exclusive lock for repository\n")
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
 	if err != nil {
-		return err
-	}
-
-	if !gopts.NoLock {
-		Verbosef("create exclusive lock for repository\n")
-		var lock *restic.Lock
-		lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
+		return nil
 	}
+	defer unlock()
 
 	changeCnt := 0
 	for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
@@ -130,6 +122,9 @@ func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []st
 			changeCnt++
 		}
 	}
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 	if changeCnt == 0 {
 		Verbosef("no snapshots were modified\n")
 	} else {

cmd/restic/cmd_tag_integration_test.go

@@ -12,6 +12,7 @@ func testRunTag(t testing.TB, opts TagOptions, gopts GlobalOptions) {
 	rtest.OK(t, runTag(context.TODO(), opts, gopts, []string{}))
 }
 
+// nolint: staticcheck // false positive nil pointer dereference check
 func TestTag(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()

cmd/restic/delete.go

@@ -1,41 +0,0 @@
-package main
-
-import (
-	"context"
-
-	"github.com/restic/restic/internal/restic"
-)
-
-// DeleteFiles deletes the given fileList of fileType in parallel
-// it will print a warning if there is an error, but continue deleting the remaining files
-func DeleteFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) {
-	_ = deleteFiles(ctx, gopts, true, repo, fileList, fileType)
-}
-
-// DeleteFilesChecked deletes the given fileList of fileType in parallel
-// if an error occurs, it will cancel and return this error
-func DeleteFilesChecked(ctx context.Context, gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
-	return deleteFiles(ctx, gopts, false, repo, fileList, fileType)
-}
-
-// deleteFiles deletes the given fileList of fileType in parallel
-// if ignoreError=true, it will print a warning if there was an error, else it will abort.
-func deleteFiles(ctx context.Context, gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
-	bar := newProgressMax(!gopts.JSON && !gopts.Quiet, 0, "files deleted")
-	defer bar.Done()
-
-	return restic.ParallelRemove(ctx, repo, fileList, fileType, func(id restic.ID, err error) error {
-		if err != nil {
-			if !gopts.JSON {
-				Warnf("unable to remove %v/%v from the repository\n", fileType, id)
-			}
-			if !ignoreError {
-				return err
-			}
-		}
-		if !gopts.JSON && gopts.verbosity > 2 {
-			Verbosef("removed %v/%v\n", fileType, id)
-		}
-		return nil
-	}, bar)
-}

cmd/restic/find.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"os"
 
 	"github.com/restic/restic/internal/restic"
 	"github.com/spf13/pflag"
@@ -14,17 +15,27 @@ func initMultiSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter,
 	if !addHostShorthand {
 		hostShorthand = ""
 	}
-	flags.StringArrayVarP(&filt.Hosts, "host", hostShorthand, nil, "only consider snapshots for this `host` (can be specified multiple times)")
+	flags.StringArrayVarP(&filt.Hosts, "host", hostShorthand, nil, "only consider snapshots for this `host` (can be specified multiple times) (default: $RESTIC_HOST)")
 	flags.Var(&filt.Tags, "tag", "only consider snapshots including `tag[,tag,...]` (can be specified multiple times)")
 	flags.StringArrayVar(&filt.Paths, "path", nil, "only consider snapshots including this (absolute) `path` (can be specified multiple times)")
+
+	// set default based on env if set
+	if host := os.Getenv("RESTIC_HOST"); host != "" {
+		filt.Hosts = []string{host}
+	}
 }
 
 // initSingleSnapshotFilter is used for commands that work on a single snapshot
 // MUST be combined with restic.FindFilteredSnapshot
 func initSingleSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter) {
-	flags.StringArrayVarP(&filt.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times)")
+	flags.StringArrayVarP(&filt.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times) (default: $RESTIC_HOST)")
 	flags.Var(&filt.Tags, "tag", "only consider snapshots including `tag[,tag,...]`, when snapshot ID \"latest\" is given (can be specified multiple times)")
 	flags.StringArrayVar(&filt.Paths, "path", nil, "only consider snapshots including this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times)")
+
+	// set default based on env if set
+	if host := os.Getenv("RESTIC_HOST"); host != "" {
+		filt.Hosts = []string{host}
+	}
 }
 
 // FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.

cmd/restic/find_test.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"testing"
+
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
+	"github.com/spf13/pflag"
+)
+
+func TestSnapshotFilter(t *testing.T) {
+	for _, test := range []struct {
+		name     string
+		args     []string
+		expected []string
+		env      string
+	}{
+		{
+			"no value",
+			[]string{},
+			nil,
+			"",
+		},
+		{
+			"args only",
+			[]string{"--host", "abc"},
+			[]string{"abc"},
+			"",
+		},
+		{
+			"env default",
+			[]string{},
+			[]string{"def"},
+			"def",
+		},
+		{
+			"both",
+			[]string{"--host", "abc"},
+			[]string{"abc"},
+			"def",
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			t.Setenv("RESTIC_HOST", test.env)
+
+			for _, mode := range []bool{false, true} {
+				set := pflag.NewFlagSet("test", pflag.PanicOnError)
+				flt := &restic.SnapshotFilter{}
+				if mode {
+					initMultiSnapshotFilter(set, flt, false)
+				} else {
+					initSingleSnapshotFilter(set, flt)
+				}
+				err := set.Parse(test.args)
+				rtest.OK(t, err)
+
+				rtest.Equals(t, test.expected, flt.Hosts, "unexpected hosts")
+			}
+		})
+	}
+}

cmd/restic/global.go

@@ -43,7 +43,7 @@ import (
 	"golang.org/x/term"
 )
 
-var version = "0.16.4-dev (compiled manually)"
+const version = "0.16.4-dev (compiled manually)"
 
 // TimeFormat is the format used for all timestamps printed by restic.
 const TimeFormat = "2006-01-02 15:04:05"
@@ -96,9 +96,6 @@ var globalOptions = GlobalOptions{
 	stderr: os.Stderr,
 }
 
-var isReadingPassword bool
-var internalGlobalCtx context.Context
-
 func init() {
 	backends := location.NewRegistry()
 	backends.Register(azure.NewFactory())
@@ -112,15 +109,6 @@ func init() {
 	backends.Register(swift.NewFactory())
 	globalOptions.backends = backends
 
-	var cancel context.CancelFunc
-	internalGlobalCtx, cancel = context.WithCancel(context.Background())
-	AddCleanupHandler(func(code int) (int, error) {
-		// Must be called before the unlock cleanup handler to ensure that the latter is
-		// not blocked due to limited number of backend connections, see #1434
-		cancel()
-		return code, nil
-	})
-
 	f := cmdRoot.PersistentFlags()
 	f.StringVarP(&globalOptions.Repo, "repo", "r", "", "`repository` to backup to or restore from (default: $RESTIC_REPOSITORY)")
 	f.StringVarP(&globalOptions.RepositoryFile, "repository-file", "", "", "`file` to read the repository location from (default: $RESTIC_REPOSITORY_FILE)")
@@ -165,8 +153,6 @@ func init() {
 	// parse target pack size from env, on error the default value will be used
 	targetPackSize, _ := strconv.ParseUint(os.Getenv("RESTIC_PACK_SIZE"), 10, 32)
 	globalOptions.PackSize = uint(targetPackSize)
-
-	restoreTerminal()
 }
 
 func stdinIsTerminal() bool {
@@ -191,40 +177,6 @@ func stdoutTerminalWidth() int {
 	return w
 }
 
-// restoreTerminal installs a cleanup handler that restores the previous
-// terminal state on exit. This handler is only intended to restore the
-// terminal configuration if restic exits after receiving a signal. A regular
-// program execution must revert changes to the terminal configuration itself.
-// The terminal configuration is only restored while reading a password.
-func restoreTerminal() {
-	if !term.IsTerminal(int(os.Stdout.Fd())) {
-		return
-	}
-
-	fd := int(os.Stdout.Fd())
-	state, err := term.GetState(fd)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "unable to get terminal state: %v\n", err)
-		return
-	}
-
-	AddCleanupHandler(func(code int) (int, error) {
-		// Restoring the terminal configuration while restic runs in the
-		// background, causes restic to get stopped on unix systems with
-		// a SIGTTOU signal. Thus only restore the terminal settings if
-		// they might have been modified, which is the case while reading
-		// a password.
-		if !isReadingPassword {
-			return code, nil
-		}
-		err := term.Restore(fd, state)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "unable to restore terminal state: %v\n", err)
-		}
-		return code, err
-	})
-}
-
 // ClearLine creates a platform dependent string to clear the current
 // line, so it can be overwritten.
 //
@@ -333,24 +285,48 @@ func readPassword(in io.Reader) (password string, err error) {
 
 // readPasswordTerminal reads the password from the given reader which must be a
 // tty. Prompt is printed on the writer out before attempting to read the
-// password.
-func readPasswordTerminal(in *os.File, out io.Writer, prompt string) (password string, err error) {
-	fmt.Fprint(out, prompt)
-	isReadingPassword = true
-	buf, err := term.ReadPassword(int(in.Fd()))
-	isReadingPassword = false
-	fmt.Fprintln(out)
+// password. If the context is canceled, the function leaks the password reading
+// goroutine.
+func readPasswordTerminal(ctx context.Context, in *os.File, out *os.File, prompt string) (password string, err error) {
+	fd := int(out.Fd())
+	state, err := term.GetState(fd)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "unable to get terminal state: %v\n", err)
+		return "", err
+	}
+
+	done := make(chan struct{})
+	var buf []byte
+
+	go func() {
+		defer close(done)
+		fmt.Fprint(out, prompt)
+		buf, err = term.ReadPassword(int(in.Fd()))
+		fmt.Fprintln(out)
+	}()
+
+	select {
+	case <-ctx.Done():
+		err := term.Restore(fd, state)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "unable to restore terminal state: %v\n", err)
+		}
+		return "", ctx.Err()
+	case <-done:
+		// clean shutdown, nothing to do
+	}
+
 	if err != nil {
 		return "", errors.Wrap(err, "ReadPassword")
 	}
 
-	password = string(buf)
-	return password, nil
+	return string(buf), nil
 }
 
 // ReadPassword reads the password from a password file, the environment
-// variable RESTIC_PASSWORD or prompts the user.
-func ReadPassword(opts GlobalOptions, prompt string) (string, error) {
+// variable RESTIC_PASSWORD or prompts the user. If the context is canceled,
+// the function leaks the password reading goroutine.
+func ReadPassword(ctx context.Context, opts GlobalOptions, prompt string) (string, error) {
 	if opts.password != "" {
 		return opts.password, nil
 	}
@@ -361,7 +337,7 @@ func ReadPassword(opts GlobalOptions, prompt string) (string, error) {
 	)
 
 	if stdinIsTerminal() {
-		password, err = readPasswordTerminal(os.Stdin, os.Stderr, prompt)
+		password, err = readPasswordTerminal(ctx, os.Stdin, os.Stderr, prompt)
 	} else {
 		password, err = readPassword(os.Stdin)
 		Verbosef("reading repository password from stdin\n")
@@ -379,14 +355,15 @@ func ReadPassword(opts GlobalOptions, prompt string) (string, error) {
 }
 
 // ReadPasswordTwice calls ReadPassword two times and returns an error when the
-// passwords don't match.
-func ReadPasswordTwice(gopts GlobalOptions, prompt1, prompt2 string) (string, error) {
-	pw1, err := ReadPassword(gopts, prompt1)
+// passwords don't match. If the context is canceled, the function leaks the
+// password reading goroutine.
+func ReadPasswordTwice(ctx context.Context, gopts GlobalOptions, prompt1, prompt2 string) (string, error) {
+	pw1, err := ReadPassword(ctx, gopts, prompt1)
 	if err != nil {
 		return "", err
 	}
 	if stdinIsTerminal() {
-		pw2, err := ReadPassword(gopts, prompt2)
+		pw2, err := ReadPassword(ctx, gopts, prompt2)
 		if err != nil {
 			return "", err
 		}
@@ -469,7 +446,10 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi
 	}
 
 	for ; passwordTriesLeft > 0; passwordTriesLeft-- {
-		opts.password, err = ReadPassword(opts, "enter password for repository: ")
+		opts.password, err = ReadPassword(ctx, opts, "enter password for repository: ")
+		if ctx.Err() != nil {
+			return nil, ctx.Err()
+		}
 		if err != nil && passwordTriesLeft > 1 {
 			opts.password = ""
 			fmt.Printf("%s. Try again\n", err)
@@ -570,16 +550,13 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
 	return cfg, nil
 }
 
-// Open the backend specified by a location config.
-func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (backend.Backend, error) {
+func innerOpen(ctx context.Context, s string, gopts GlobalOptions, opts options.Options, create bool) (backend.Backend, error) {
 	debug.Log("parsing location %v", location.StripPassword(gopts.backends, s))
 	loc, err := location.Parse(gopts.backends, s)
 	if err != nil {
 		return nil, errors.Fatalf("parsing repository location failed: %v", err)
 	}
 
-	var be backend.Backend
-
 	cfg, err := parseConfig(loc, opts)
 	if err != nil {
 		return nil, err
@@ -599,7 +576,13 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio
 		return nil, errors.Fatalf("invalid backend: %q", loc.Scheme)
 	}
 
-	be, err = factory.Open(ctx, cfg, rt, lim)
+	var be backend.Backend
+	if create {
+		be, err = factory.Create(ctx, cfg, rt, lim)
+	} else {
+		be, err = factory.Open(ctx, cfg, rt, lim)
+	}
+
 	if err != nil {
 		return nil, errors.Fatalf("unable to open repository at %v: %v", location.StripPassword(gopts.backends, s), err)
 	}
@@ -615,6 +598,17 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio
 		}
 	}
 
+	return be, nil
+}
+
+// Open the backend specified by a location config.
+func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (backend.Backend, error) {
+
+	be, err := innerOpen(ctx, s, gopts, opts, false)
+	if err != nil {
+		return nil, err
+	}
+
 	// check if config is there
 	fi, err := be.Stat(ctx, backend.Handle{Type: restic.ConfigFile})
 	if err != nil {
@@ -630,31 +624,5 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio
 
 // Create the backend specified by URI.
 func create(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (backend.Backend, error) {
-	debug.Log("parsing location %v", location.StripPassword(gopts.backends, s))
-	loc, err := location.Parse(gopts.backends, s)
-	if err != nil {
-		return nil, err
-	}
-
-	cfg, err := parseConfig(loc, opts)
-	if err != nil {
-		return nil, err
-	}
-
-	rt, err := backend.Transport(globalOptions.TransportOptions)
-	if err != nil {
-		return nil, errors.Fatal(err.Error())
-	}
-
-	factory := gopts.backends.Lookup(loc.Scheme)
-	if factory == nil {
-		return nil, errors.Fatalf("invalid backend: %q", loc.Scheme)
-	}
-
-	be, err := factory.Create(ctx, cfg, rt, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	return logger.New(sema.NewBackend(be)), nil
+	return innerOpen(ctx, s, gopts, opts, true)
 }

cmd/restic/global_debug.go

@@ -15,23 +15,28 @@ import (
 	"github.com/pkg/profile"
 )
 
-var (
-	listenProfile    string
-	memProfilePath   string
-	cpuProfilePath   string
-	traceProfilePath string
-	blockProfilePath string
-	insecure         bool
-)
+type ProfileOptions struct {
+	listen    string
+	memPath   string
+	cpuPath   string
+	tracePath string
+	blockPath string
+	insecure  bool
+}
+
+var profileOpts ProfileOptions
+var prof interface {
+	Stop()
+}
 
 func init() {
 	f := cmdRoot.PersistentFlags()
-	f.StringVar(&listenProfile, "listen-profile", "", "listen on this `address:port` for memory profiling")
-	f.StringVar(&memProfilePath, "mem-profile", "", "write memory profile to `dir`")
-	f.StringVar(&cpuProfilePath, "cpu-profile", "", "write cpu profile to `dir`")
-	f.StringVar(&traceProfilePath, "trace-profile", "", "write trace to `dir`")
-	f.StringVar(&blockProfilePath, "block-profile", "", "write block profile to `dir`")
-	f.BoolVar(&insecure, "insecure-kdf", false, "use insecure KDF settings")
+	f.StringVar(&profileOpts.listen, "listen-profile", "", "listen on this `address:port` for memory profiling")
+	f.StringVar(&profileOpts.memPath, "mem-profile", "", "write memory profile to `dir`")
+	f.StringVar(&profileOpts.cpuPath, "cpu-profile", "", "write cpu profile to `dir`")
+	f.StringVar(&profileOpts.tracePath, "trace-profile", "", "write trace to `dir`")
+	f.StringVar(&profileOpts.blockPath, "block-profile", "", "write block profile to `dir`")
+	f.BoolVar(&profileOpts.insecure, "insecure-kdf", false, "use insecure KDF settings")
 }
 
 type fakeTestingTB struct{}
@@ -41,10 +46,10 @@ func (fakeTestingTB) Logf(msg string, args ...interface{}) {
 }
 
 func runDebug() error {
-	if listenProfile != "" {
-		fmt.Fprintf(os.Stderr, "running profile HTTP server on %v\n", listenProfile)
+	if profileOpts.listen != "" {
+		fmt.Fprintf(os.Stderr, "running profile HTTP server on %v\n", profileOpts.listen)
 		go func() {
-			err := http.ListenAndServe(listenProfile, nil)
+			err := http.ListenAndServe(profileOpts.listen, nil)
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "profile HTTP server listen failed: %v\n", err)
 			}
@@ -52,16 +57,16 @@ func runDebug() error {
 	}
 
 	profilesEnabled := 0
-	if memProfilePath != "" {
+	if profileOpts.memPath != "" {
 		profilesEnabled++
 	}
-	if cpuProfilePath != "" {
+	if profileOpts.cpuPath != "" {
 		profilesEnabled++
 	}
-	if traceProfilePath != "" {
+	if profileOpts.tracePath != "" {
 		profilesEnabled++
 	}
-	if blockProfilePath != "" {
+	if profileOpts.blockPath != "" {
 		profilesEnabled++
 	}
 
@@ -69,30 +74,25 @@ func runDebug() error {
 		return errors.Fatal("only one profile (memory, CPU, trace, or block) may be activated at the same time")
 	}
 
-	var prof interface {
-		Stop()
+	if profileOpts.memPath != "" {
+		prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.MemProfile, profile.ProfilePath(profileOpts.memPath))
+	} else if profileOpts.cpuPath != "" {
+		prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.CPUProfile, profile.ProfilePath(profileOpts.cpuPath))
+	} else if profileOpts.tracePath != "" {
+		prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.TraceProfile, profile.ProfilePath(profileOpts.tracePath))
+	} else if profileOpts.blockPath != "" {
+		prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.BlockProfile, profile.ProfilePath(profileOpts.blockPath))
 	}
 
-	if memProfilePath != "" {
-		prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.MemProfile, profile.ProfilePath(memProfilePath))
-	} else if cpuProfilePath != "" {
-		prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.CPUProfile, profile.ProfilePath(cpuProfilePath))
-	} else if traceProfilePath != "" {
-		prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.TraceProfile, profile.ProfilePath(traceProfilePath))
-	} else if blockProfilePath != "" {
-		prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.BlockProfile, profile.ProfilePath(blockProfilePath))
-	}
-
-	if prof != nil {
-		AddCleanupHandler(func(code int) (int, error) {
-			prof.Stop()
-			return code, nil
-		})
-	}
-
-	if insecure {
+	if profileOpts.insecure {
 		repository.TestUseLowSecurityKDFParameters(fakeTestingTB{})
 	}
 
 	return nil
 }
+
+func stopDebug() {
+	if prof != nil {
+		prof.Stop()
+	}
+}

cmd/restic/global_release.go

@@ -5,3 +5,6 @@ package main
 
 // runDebug is a noop without the debug tag.
 func runDebug() error { return nil }
+
+// stopDebug is a noop without the debug tag.
+func stopDebug() {}

cmd/restic/integration_helpers_test.go

@@ -232,47 +232,66 @@ func testSetupBackupData(t testing.TB, env *testEnvironment) string {
 }
 
 func listPacks(gopts GlobalOptions, t *testing.T) restic.IDSet {
-	r, err := OpenRepository(context.TODO(), gopts)
+	ctx, r, unlock, err := openWithReadLock(context.TODO(), gopts, false)
 	rtest.OK(t, err)
+	defer unlock()
 
 	packs := restic.NewIDSet()
 
-	rtest.OK(t, r.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
+	rtest.OK(t, r.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
 		packs.Insert(id)
 		return nil
 	}))
 	return packs
 }
 
-func removePacks(gopts GlobalOptions, t testing.TB, remove restic.IDSet) {
-	r, err := OpenRepository(context.TODO(), gopts)
+func listTreePacks(gopts GlobalOptions, t *testing.T) restic.IDSet {
+	ctx, r, unlock, err := openWithReadLock(context.TODO(), gopts, false)
 	rtest.OK(t, err)
+	defer unlock()
+
+	rtest.OK(t, r.LoadIndex(ctx, nil))
+	treePacks := restic.NewIDSet()
+	rtest.OK(t, r.Index().Each(ctx, func(pb restic.PackedBlob) {
+		if pb.Type == restic.TreeBlob {
+			treePacks.Insert(pb.PackID)
+		}
+	}))
+
+	return treePacks
+}
+
+func removePacks(gopts GlobalOptions, t testing.TB, remove restic.IDSet) {
+	ctx, r, unlock, err := openWithExclusiveLock(context.TODO(), gopts, false)
+	rtest.OK(t, err)
+	defer unlock()
 
 	for id := range remove {
-		rtest.OK(t, r.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()}))
+		rtest.OK(t, r.Backend().Remove(ctx, backend.Handle{Type: restic.PackFile, Name: id.String()}))
 	}
 }
 
 func removePacksExcept(gopts GlobalOptions, t testing.TB, keep restic.IDSet, removeTreePacks bool) {
-	r, err := OpenRepository(context.TODO(), gopts)
+	ctx, r, unlock, err := openWithExclusiveLock(context.TODO(), gopts, false)
 	rtest.OK(t, err)
+	defer unlock()
 
 	// Get all tree packs
-	rtest.OK(t, r.LoadIndex(context.TODO(), nil))
+	rtest.OK(t, r.LoadIndex(ctx, nil))
 
 	treePacks := restic.NewIDSet()
-	r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
+	rtest.OK(t, r.Index().Each(ctx, func(pb restic.PackedBlob) {
 		if pb.Type == restic.TreeBlob {
 			treePacks.Insert(pb.PackID)
 		}
-	})
+	}))
 
 	// remove all packs containing data blobs
-	rtest.OK(t, r.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
+	rtest.OK(t, r.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
 		if treePacks.Has(id) != removeTreePacks || keep.Has(id) {
 			return nil
 		}
-		return r.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: id.String()})
+		return r.Backend().Remove(ctx, backend.Handle{Type: restic.PackFile, Name: id.String()})
 	}))
 }
 

cmd/restic/integration_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
+	"github.com/restic/restic/internal/ui/termstatus"
 )
 
 func TestCheckRestoreNoLock(t *testing.T) {
@@ -88,8 +89,12 @@ func TestListOnce(t *testing.T) {
 	testRunPrune(t, env.gopts, pruneOpts)
 	rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil))
 
-	rtest.OK(t, runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts))
-	rtest.OK(t, runRebuildIndex(context.TODO(), RepairIndexOptions{ReadAllPacks: true}, env.gopts))
+	rtest.OK(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+		return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts, term)
+	}))
+	rtest.OK(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+		return runRebuildIndex(context.TODO(), RepairIndexOptions{ReadAllPacks: true}, env.gopts, term)
+	}))
 }
 
 type writeToOnly struct {
@@ -154,12 +159,13 @@ func TestFindListOnce(t *testing.T) {
 	testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "3")}, opts, env.gopts)
 	thirdSnapshot := restic.NewIDSet(testListSnapshots(t, env.gopts, 3)...)
 
-	repo, err := OpenRepository(context.TODO(), env.gopts)
+	ctx, repo, unlock, err := openWithReadLock(context.TODO(), env.gopts, false)
 	rtest.OK(t, err)
+	defer unlock()
 
 	snapshotIDs := restic.NewIDSet()
 	// specify the two oldest snapshots explicitly and use "latest" to reference the newest one
-	for sn := range FindFilteredSnapshots(context.TODO(), repo, repo, &restic.SnapshotFilter{}, []string{
+	for sn := range FindFilteredSnapshots(ctx, repo, repo, &restic.SnapshotFilter{}, []string{
 		secondSnapshot[0].String(),
 		secondSnapshot[1].String()[:8],
 		"latest",

cmd/restic/lock.go

@@ -2,316 +2,47 @@ package main
 
 import (
 	"context"
-	"fmt"
-	"sync"
-	"time"
 
-	"github.com/restic/restic/internal/backend"
-	"github.com/restic/restic/internal/debug"
-	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/repository"
 )
 
-type lockContext struct {
-	lock      *restic.Lock
-	cancel    context.CancelFunc
-	refreshWG sync.WaitGroup
-}
-
-var globalLocks struct {
-	locks map[*restic.Lock]*lockContext
-	sync.Mutex
-	sync.Once
-}
-
-func lockRepo(ctx context.Context, repo restic.Repository, retryLock time.Duration, json bool) (*restic.Lock, context.Context, error) {
-	return lockRepository(ctx, repo, false, retryLock, json)
-}
-
-func lockRepoExclusive(ctx context.Context, repo restic.Repository, retryLock time.Duration, json bool) (*restic.Lock, context.Context, error) {
-	return lockRepository(ctx, repo, true, retryLock, json)
-}
-
-var (
-	retrySleepStart = 5 * time.Second
-	retrySleepMax   = 60 * time.Second
-)
-
-func minDuration(a, b time.Duration) time.Duration {
-	if a <= b {
-		return a
-	}
-	return b
-}
-
-// lockRepository wraps the ctx such that it is cancelled when the repository is unlocked
-// cancelling the original context also stops the lock refresh
-func lockRepository(ctx context.Context, repo restic.Repository, exclusive bool, retryLock time.Duration, json bool) (*restic.Lock, context.Context, error) {
-	// make sure that a repository is unlocked properly and after cancel() was
-	// called by the cleanup handler in global.go
-	globalLocks.Do(func() {
-		AddCleanupHandler(unlockAll)
-	})
-
-	lockFn := restic.NewLock
-	if exclusive {
-		lockFn = restic.NewExclusiveLock
-	}
-
-	var lock *restic.Lock
-	var err error
-
-	retrySleep := minDuration(retrySleepStart, retryLock)
-	retryMessagePrinted := false
-	retryTimeout := time.After(retryLock)
-
-retryLoop:
-	for {
-		lock, err = lockFn(ctx, repo)
-		if err != nil && restic.IsAlreadyLocked(err) {
-
-			if !retryMessagePrinted {
-				if !json {
-					Verbosef("repo already locked, waiting up to %s for the lock\n", retryLock)
-				}
-				retryMessagePrinted = true
-			}
-
-			debug.Log("repo already locked, retrying in %v", retrySleep)
-			retrySleepCh := time.After(retrySleep)
-
-			select {
-			case <-ctx.Done():
-				return nil, ctx, ctx.Err()
-			case <-retryTimeout:
-				debug.Log("repo already locked, timeout expired")
-				// Last lock attempt
-				lock, err = lockFn(ctx, repo)
-				break retryLoop
-			case <-retrySleepCh:
-				retrySleep = minDuration(retrySleep*2, retrySleepMax)
-			}
-		} else {
-			// anything else, either a successful lock or another error
-			break retryLoop
-		}
-	}
-	if restic.IsInvalidLock(err) {
-		return nil, ctx, errors.Fatalf("%v\n\nthe `unlock --remove-all` command can be used to remove invalid locks. Make sure that no other restic process is accessing the repository when running the command", err)
-	}
+func internalOpenWithLocked(ctx context.Context, gopts GlobalOptions, dryRun bool, exclusive bool) (context.Context, *repository.Repository, func(), error) {
+	repo, err := OpenRepository(ctx, gopts)
 	if err != nil {
-		return nil, ctx, fmt.Errorf("unable to create lock in backend: %w", err)
+		return nil, nil, nil, err
 	}
-	debug.Log("create lock %p (exclusive %v)", lock, exclusive)
 
-	ctx, cancel := context.WithCancel(ctx)
-	lockInfo := &lockContext{
-		lock:   lock,
-		cancel: cancel,
-	}
-	lockInfo.refreshWG.Add(2)
-	refreshChan := make(chan struct{})
-	forceRefreshChan := make(chan refreshLockRequest)
+	unlock := func() {}
+	if !dryRun {
+		var lock *repository.Unlocker
 
-	globalLocks.Lock()
-	globalLocks.locks[lock] = lockInfo
-	go refreshLocks(ctx, repo.Backend(), lockInfo, refreshChan, forceRefreshChan)
-	go monitorLockRefresh(ctx, lockInfo, refreshChan, forceRefreshChan)
-	globalLocks.Unlock()
-
-	return lock, ctx, err
-}
-
-var refreshInterval = 5 * time.Minute
-
-// consider a lock refresh failed a bit before the lock actually becomes stale
-// the difference allows to compensate for a small time drift between clients.
-var refreshabilityTimeout = restic.StaleLockTimeout - refreshInterval*3/2
-
-type refreshLockRequest struct {
-	result chan bool
-}
-
-func refreshLocks(ctx context.Context, backend backend.Backend, lockInfo *lockContext, refreshed chan<- struct{}, forceRefresh <-chan refreshLockRequest) {
-	debug.Log("start")
-	lock := lockInfo.lock
-	ticker := time.NewTicker(refreshInterval)
-	lastRefresh := lock.Time
-
-	defer func() {
-		ticker.Stop()
-		// ensure that the context was cancelled before removing the lock
-		lockInfo.cancel()
-
-		// remove the lock from the repo
-		debug.Log("unlocking repository with lock %v", lock)
-		if err := lock.Unlock(); err != nil {
-			debug.Log("error while unlocking: %v", err)
-			Warnf("error while unlocking: %v", err)
+		lock, ctx, err = repository.Lock(ctx, repo, exclusive, gopts.RetryLock, func(msg string) {
+			if !gopts.JSON {
+				Verbosef("%s", msg)
+			}
+		}, Warnf)
+		if err != nil {
+			return nil, nil, nil, err
 		}
 
-		lockInfo.refreshWG.Done()
-	}()
-
-	for {
-		select {
-		case <-ctx.Done():
-			debug.Log("terminate")
-			return
-
-		case req := <-forceRefresh:
-			debug.Log("trying to refresh stale lock")
-			// keep on going if our current lock still exists
-			success := tryRefreshStaleLock(ctx, backend, lock, lockInfo.cancel)
-			// inform refresh goroutine about forced refresh
-			select {
-			case <-ctx.Done():
-			case req.result <- success:
-			}
-
-			if success {
-				// update lock refresh time
-				lastRefresh = lock.Time
-			}
-
-		case <-ticker.C:
-			if time.Since(lastRefresh) > refreshabilityTimeout {
-				// the lock is too old, wait until the expiry monitor cancels the context
-				continue
-			}
-
-			debug.Log("refreshing locks")
-			err := lock.Refresh(context.TODO())
-			if err != nil {
-				Warnf("unable to refresh lock: %v\n", err)
-			} else {
-				lastRefresh = lock.Time
-				// inform monitor goroutine about successful refresh
-				select {
-				case <-ctx.Done():
-				case refreshed <- struct{}{}:
-				}
-			}
-		}
+		unlock = lock.Unlock
+	} else {
+		repo.SetDryRun()
 	}
+
+	return ctx, repo, unlock, nil
 }
 
-func monitorLockRefresh(ctx context.Context, lockInfo *lockContext, refreshed <-chan struct{}, forceRefresh chan<- refreshLockRequest) {
-	// time.Now() might use a monotonic timer which is paused during standby
-	// convert to unix time to ensure we compare real time values
-	lastRefresh := time.Now().UnixNano()
-	pollDuration := 1 * time.Second
-	if refreshInterval < pollDuration {
-		// require for TestLockFailedRefresh
-		pollDuration = refreshInterval / 5
-	}
-	// timers are paused during standby, which is a problem as the refresh timeout
-	// _must_ expire if the host was too long in standby. Thus fall back to periodic checks
-	// https://github.com/golang/go/issues/35012
-	ticker := time.NewTicker(pollDuration)
-	defer func() {
-		ticker.Stop()
-		lockInfo.cancel()
-		lockInfo.refreshWG.Done()
-	}()
-
-	var refreshStaleLockResult chan bool
-
-	for {
-		select {
-		case <-ctx.Done():
-			debug.Log("terminate expiry monitoring")
-			return
-		case <-refreshed:
-			if refreshStaleLockResult != nil {
-				// ignore delayed refresh notifications while the stale lock is refreshed
-				continue
-			}
-			lastRefresh = time.Now().UnixNano()
-		case <-ticker.C:
-			if time.Now().UnixNano()-lastRefresh < refreshabilityTimeout.Nanoseconds() || refreshStaleLockResult != nil {
-				continue
-			}
-
-			debug.Log("trying to refreshStaleLock")
-			// keep on going if our current lock still exists
-			refreshReq := refreshLockRequest{
-				result: make(chan bool),
-			}
-			refreshStaleLockResult = refreshReq.result
-
-			// inform refresh goroutine about forced refresh
-			select {
-			case <-ctx.Done():
-			case forceRefresh <- refreshReq:
-			}
-		case success := <-refreshStaleLockResult:
-			if success {
-				lastRefresh = time.Now().UnixNano()
-				refreshStaleLockResult = nil
-				continue
-			}
-
-			Warnf("Fatal: failed to refresh lock in time\n")
-			return
-		}
-	}
+func openWithReadLock(ctx context.Context, gopts GlobalOptions, noLock bool) (context.Context, *repository.Repository, func(), error) {
+	// TODO enfore read-only operations once the locking code has moved to the repository
+	return internalOpenWithLocked(ctx, gopts, noLock, false)
 }
 
-func tryRefreshStaleLock(ctx context.Context, be backend.Backend, lock *restic.Lock, cancel context.CancelFunc) bool {
-	freeze := backend.AsBackend[backend.FreezeBackend](be)
-	if freeze != nil {
-		debug.Log("freezing backend")
-		freeze.Freeze()
-		defer freeze.Unfreeze()
-	}
-
-	err := lock.RefreshStaleLock(ctx)
-	if err != nil {
-		Warnf("failed to refresh stale lock: %v\n", err)
-		// cancel context while the backend is still frozen to prevent accidental modifications
-		cancel()
-		return false
-	}
-
-	return true
+func openWithAppendLock(ctx context.Context, gopts GlobalOptions, dryRun bool) (context.Context, *repository.Repository, func(), error) {
+	// TODO enfore non-exclusive operations once the locking code has moved to the repository
+	return internalOpenWithLocked(ctx, gopts, dryRun, false)
 }
 
-func unlockRepo(lock *restic.Lock) {
-	if lock == nil {
-		return
-	}
-
-	globalLocks.Lock()
-	lockInfo, exists := globalLocks.locks[lock]
-	delete(globalLocks.locks, lock)
-	globalLocks.Unlock()
-
-	if !exists {
-		debug.Log("unable to find lock %v in the global list of locks, ignoring", lock)
-		return
-	}
-	lockInfo.cancel()
-	lockInfo.refreshWG.Wait()
-}
-
-func unlockAll(code int) (int, error) {
-	globalLocks.Lock()
-	locks := globalLocks.locks
-	debug.Log("unlocking %d locks", len(globalLocks.locks))
-	for _, lockInfo := range globalLocks.locks {
-		lockInfo.cancel()
-	}
-	globalLocks.locks = make(map[*restic.Lock]*lockContext)
-	globalLocks.Unlock()
-
-	for _, lockInfo := range locks {
-		lockInfo.refreshWG.Wait()
-	}
-
-	return code, nil
-}
-
-func init() {
-	globalLocks.locks = make(map[*restic.Lock]*lockContext)
+func openWithExclusiveLock(ctx context.Context, gopts GlobalOptions, dryRun bool) (context.Context, *repository.Repository, func(), error) {
+	return internalOpenWithLocked(ctx, gopts, dryRun, true)
 }

cmd/restic/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bufio"
 	"bytes"
+	"context"
 	"fmt"
 	"log"
 	"os"
@@ -24,6 +25,8 @@ func init() {
 	_, _ = maxprocs.Set()
 }
 
+var ErrOK = errors.New("ok")
+
 // cmdRoot is the base command when no other command has been specified.
 var cmdRoot = &cobra.Command{
 	Use:   "restic",
@@ -74,6 +77,9 @@ The full documentation can be found at https://restic.readthedocs.io/ .
 		// enabled)
 		return runDebug()
 	},
+	PersistentPostRun: func(_ *cobra.Command, _ []string) {
+		stopDebug()
+	},
 }
 
 // Distinguish commands that need the password from those that work without,
@@ -88,8 +94,6 @@ func needsPassword(cmd string) bool {
 	}
 }
 
-var logBuffer = bytes.NewBuffer(nil)
-
 func tweakGoGC() {
 	// lower GOGC from 100 to 50, unless it was manually overwritten by the user
 	oldValue := godebug.SetGCPercent(50)
@@ -102,6 +106,7 @@ func main() {
 	tweakGoGC()
 	// install custom global logger into a buffer, if an error occurs
 	// we can show the logs
+	logBuffer := bytes.NewBuffer(nil)
 	log.SetOutput(logBuffer)
 
 	err := feature.Flag.Apply(os.Getenv("RESTIC_FEATURES"), func(s string) {
@@ -115,7 +120,16 @@ func main() {
 	debug.Log("main %#v", os.Args)
 	debug.Log("restic %s compiled with %v on %v/%v",
 		version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
-	err = cmdRoot.ExecuteContext(internalGlobalCtx)
+
+	ctx := createGlobalContext()
+	err = cmdRoot.ExecuteContext(ctx)
+
+	if err == nil {
+		err = ctx.Err()
+	} else if err == ErrOK {
+		// ErrOK overwrites context cancelation errors
+		err = nil
+	}
 
 	switch {
 	case restic.IsAlreadyLocked(err):
@@ -137,11 +151,13 @@ func main() {
 	}
 
 	var exitCode int
-	switch err {
-	case nil:
+	switch {
+	case err == nil:
 		exitCode = 0
-	case ErrInvalidSourceData:
+	case err == ErrInvalidSourceData:
 		exitCode = 3
+	case errors.Is(err, context.Canceled):
+		exitCode = 130
 	default:
 		exitCode = 1
 	}

cmd/restic/secondary_repo.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"os"
 
 	"github.com/restic/restic/internal/errors"
@@ -56,7 +57,7 @@ func initSecondaryRepoOptions(f *pflag.FlagSet, opts *secondaryRepoOptions, repo
 	opts.PasswordCommand = os.Getenv("RESTIC_FROM_PASSWORD_COMMAND")
 }
 
-func fillSecondaryGlobalOpts(opts secondaryRepoOptions, gopts GlobalOptions, repoPrefix string) (GlobalOptions, bool, error) {
+func fillSecondaryGlobalOpts(ctx context.Context, opts secondaryRepoOptions, gopts GlobalOptions, repoPrefix string) (GlobalOptions, bool, error) {
 	if opts.Repo == "" && opts.RepositoryFile == "" && opts.LegacyRepo == "" && opts.LegacyRepositoryFile == "" {
 		return GlobalOptions{}, false, errors.Fatal("Please specify a source repository location (--from-repo or --from-repository-file)")
 	}
@@ -109,7 +110,7 @@ func fillSecondaryGlobalOpts(opts secondaryRepoOptions, gopts GlobalOptions, rep
 			return GlobalOptions{}, false, err
 		}
 	}
-	dstGopts.password, err = ReadPassword(dstGopts, "enter password for "+repoPrefix+" repository: ")
+	dstGopts.password, err = ReadPassword(ctx, dstGopts, "enter password for "+repoPrefix+" repository: ")
 	if err != nil {
 		return GlobalOptions{}, false, err
 	}

cmd/restic/secondary_repo_test.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"os"
 	"path/filepath"
 	"testing"
@@ -170,7 +171,7 @@ func TestFillSecondaryGlobalOpts(t *testing.T) {
 
 	// Test all valid cases
 	for _, testCase := range validSecondaryRepoTestCases {
-		DstGOpts, isFromRepo, err := fillSecondaryGlobalOpts(testCase.Opts, gOpts, "destination")
+		DstGOpts, isFromRepo, err := fillSecondaryGlobalOpts(context.TODO(), testCase.Opts, gOpts, "destination")
 		rtest.OK(t, err)
 		rtest.Equals(t, DstGOpts, testCase.DstGOpts)
 		rtest.Equals(t, isFromRepo, testCase.FromRepo)
@@ -178,7 +179,7 @@ func TestFillSecondaryGlobalOpts(t *testing.T) {
 
 	// Test all invalid cases
 	for _, testCase := range invalidSecondaryRepoTestCases {
-		_, _, err := fillSecondaryGlobalOpts(testCase.Opts, gOpts, "destination")
+		_, _, err := fillSecondaryGlobalOpts(context.TODO(), testCase.Opts, gOpts, "destination")
 		rtest.Assert(t, err != nil, "Expected error, but function did not return an error")
 	}
 }

doc/020_installation.rst

@@ -362,3 +362,18 @@ Example for using sudo to write a zsh completion script directly to the system-w
    the operating system used, e.g. ``/usr/share/bash-completion/completions/restic``
    in Debian and derivatives. Please look up the correct path in the appropriate
    documentation.
+
+Example for setting up a powershell completion script for the local user's profile:
+
+.. code-block:: pwsh-session
+
+    # Create profile if one does not exist
+    PS> If (!(Test-Path $PROFILE.CurrentUserAllHosts)) {New-Item -Path $PROFILE.CurrentUserAllHosts -Force}
+
+    PS> $ProfileDir = (Get-Item $PROFILE.CurrentUserAllHosts).Directory
+
+    # Generate Restic completions in the same directory as the profile
+    PS> restic generate --powershell-completion "$ProfileDir\restic-completion.ps1"
+
+    # Append to the profile file the command to load Restic completions
+    PS> Add-Content -Path $PROFILE.CurrentUserAllHosts -Value "`r`nImport-Module $ProfileDir\restic-completion.ps1"

doc/030_preparing_a_new_repo.rst

@@ -201,15 +201,16 @@ scheme like this:
     $ restic -r rest:http://host:8000/ init
 
 Depending on your REST server setup, you can use HTTPS protocol,
-password protection, multiple repositories or any combination of
-those features. The TCP/IP port is also configurable. Here
-are some more examples:
+unix socket, password protection, multiple repositories or any
+combination of those features. The TCP/IP port is also configurable.
+Here are some more examples:
 
 .. code-block:: console
 
     $ restic -r rest:https://host:8000/ init
     $ restic -r rest:https://user:pass@host:8000/ init
     $ restic -r rest:https://user:pass@host:8000/my_backup_repo/ init
+    $ restic -r rest:http+unix:///tmp/rest.socket:/my_backup_repo/ init
 
 The server username and password can be specified using environment
 variables as well:

doc/045_working_with_repos.rst

@@ -18,19 +18,21 @@ Working with repositories
 Listing all snapshots
 =====================
 
-Now, you can list all the snapshots stored in the repository:
+Now, you can list all the snapshots stored in the repository. The size column
+only exists for snapshots created using restic 0.17.0 or later. It reflects the
+size of the contained files at the time when the snapshot was created.
 
 .. code-block:: console
 
     $ restic -r /srv/restic-repo snapshots
     enter password for repository:
-    ID        Date                 Host    Tags   Directory
-    ----------------------------------------------------------------------
-    40dc1520  2015-05-08 21:38:30  kasimir        /home/user/work
-    79766175  2015-05-08 21:40:19  kasimir        /home/user/work
-    bdbd3439  2015-05-08 21:45:17  luigi          /home/art
-    590c8fc8  2015-05-08 21:47:38  kazik          /srv
-    9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+    ID        Date                 Host    Tags   Directory        Size
+    -------------------------------------------------------------------------
+    40dc1520  2015-05-08 21:38:30  kasimir        /home/user/work  20.643GiB
+    79766175  2015-05-08 21:40:19  kasimir        /home/user/work  20.645GiB
+    bdbd3439  2015-05-08 21:45:17  luigi          /home/art        3.141GiB
+    590c8fc8  2015-05-08 21:47:38  kazik          /srv             580.200MiB
+    9f0bc19e  2015-05-08 21:46:11  luigi          /srv             572.180MiB
 
 You can filter the listing by directory path:
 
@@ -38,10 +40,10 @@ You can filter the listing by directory path:
 
     $ restic -r /srv/restic-repo snapshots --path="/srv"
     enter password for repository:
-    ID        Date                 Host    Tags   Directory
-    ----------------------------------------------------------------------
-    590c8fc8  2015-05-08 21:47:38  kazik          /srv
-    9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+    ID        Date                 Host    Tags   Directory  Size
+    -------------------------------------------------------------------
+    590c8fc8  2015-05-08 21:47:38  kazik          /srv       580.200MiB
+    9f0bc19e  2015-05-08 21:46:11  luigi          /srv       572.180MiB
 
 Or filter by host:
 
@@ -49,10 +51,10 @@ Or filter by host:
 
     $ restic -r /srv/restic-repo snapshots --host luigi
     enter password for repository:
-    ID        Date                 Host    Tags   Directory
-    ----------------------------------------------------------------------
-    bdbd3439  2015-05-08 21:45:17  luigi          /home/art
-    9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+    ID        Date                 Host    Tags   Directory  Size
+    -------------------------------------------------------------------
+    bdbd3439  2015-05-08 21:45:17  luigi          /home/art  3.141GiB
+    9f0bc19e  2015-05-08 21:46:11  luigi          /srv       572.180MiB
 
 Combining filters is also possible.
 
@@ -64,21 +66,21 @@ Furthermore you can group the output by the same filters (host, paths, tags):
 
     enter password for repository:
     snapshots for (host [kasimir])
-    ID        Date                 Host    Tags   Directory
-    ----------------------------------------------------------------------
-    40dc1520  2015-05-08 21:38:30  kasimir        /home/user/work
-    79766175  2015-05-08 21:40:19  kasimir        /home/user/work
+    ID        Date                 Host    Tags   Directory        Size
+    ------------------------------------------------------------------------
+    40dc1520  2015-05-08 21:38:30  kasimir        /home/user/work  20.643GiB
+    79766175  2015-05-08 21:40:19  kasimir        /home/user/work  20.645GiB
     2 snapshots
     snapshots for (host [luigi])
-    ID        Date                 Host    Tags   Directory
-    ----------------------------------------------------------------------
-    bdbd3439  2015-05-08 21:45:17  luigi          /home/art
-    9f0bc19e  2015-05-08 21:46:11  luigi          /srv
+    ID        Date                 Host    Tags   Directory  Size
+    -------------------------------------------------------------------
+    bdbd3439  2015-05-08 21:45:17  luigi          /home/art  3.141GiB
+    9f0bc19e  2015-05-08 21:46:11  luigi          /srv       572.180MiB
     2 snapshots
     snapshots for (host [kazik])
-    ID        Date                 Host    Tags   Directory
-    ----------------------------------------------------------------------
-    590c8fc8  2015-05-08 21:47:38  kazik          /srv
+    ID        Date                 Host    Tags   Directory  Size
+    -------------------------------------------------------------------
+    590c8fc8  2015-05-08 21:47:38  kazik          /srv       580.200MiB
     1 snapshots
 
 

doc/047_tuning_backup_parameters.rst

@@ -121,7 +121,7 @@ Feature flags allow disabling or enabling certain experimental restic features.
 can be specified via the ``RESTIC_FEATURES`` environment variable. The variable expects a
 comma-separated list of ``key[=value],key2[=value2]`` pairs. The key is the name of a feature
 flag. The value is optional and can contain either the value ``true`` (default if omitted)
-or ``false``. The list of currently available feautre flags is shown by the ``features``
+or ``false``. The list of currently available feature flags is shown by the ``features``
 command.
 
 Restic will return an error if an invalid feature flag is specified. No longer relevant

doc/075_scripting.rst

@@ -163,7 +163,9 @@ Summary is the last output line in a successful backup.
 +---------------------------+---------------------------------------------------------+
 | ``tree_blobs``            | Number of tree blobs                                    |
 +---------------------------+---------------------------------------------------------+
-| ``data_added``            | Amount of data added, in bytes                          |
+| ``data_added``            | Amount of (uncompressed) data added, in bytes           |
++---------------------------+---------------------------------------------------------+
+| ``data_added_packed``     | Amount of data added (after compression), in bytes      |
 +---------------------------+---------------------------------------------------------+
 | ``total_files_processed`` | Total number of files processed                         |
 +---------------------------+---------------------------------------------------------+
@@ -365,13 +367,13 @@ Snapshot object
 
 Reason object
 
-+----------------+---------------------------------------------------------+
-| ``snapshot``   | Snapshot object, without ``id`` and ``short_id`` fields |
-+----------------+---------------------------------------------------------+
-| ``matches``    | Array containing descriptions of the matching criteria  |
-+----------------+---------------------------------------------------------+
-| ``counters``   | Object containing counters used by the policies         |
-+----------------+---------------------------------------------------------+
++----------------+-----------------------------------------------------------+
+| ``snapshot``   | Snapshot object, including ``id`` and ``short_id`` fields |
++----------------+-----------------------------------------------------------+
+| ``matches``    | Array containing descriptions of the matching criteria    |
++----------------+-----------------------------------------------------------+
+| ``counters``   | Object containing counters used by the policies           |
++----------------+-----------------------------------------------------------+
 
 
 init
@@ -551,11 +553,48 @@ The snapshots command returns a single JSON object, an array with objects of the
 +---------------------+--------------------------------------------------+
 | ``program_version`` | restic version used to create snapshot           |
 +---------------------+--------------------------------------------------+
+| ``summary``         | Snapshot statistics, see "Summary object"        |
++---------------------+--------------------------------------------------+
 | ``id``              | Snapshot ID                                      |
 +---------------------+--------------------------------------------------+
 | ``short_id``        | Snapshot ID, short form                          |
 +---------------------+--------------------------------------------------+
 
+Summary object
+
+The contained statistics reflect the information at the point in time when the snapshot
+was created.
+
++---------------------------+---------------------------------------------------------+
+| ``backup_start``          | Time at which the backup was started                    |
++---------------------------+---------------------------------------------------------+
+| ``backup_end``            | Time at which the backup was completed                  |
++---------------------------+---------------------------------------------------------+
+| ``files_new``             | Number of new files                                     |
++---------------------------+---------------------------------------------------------+
+| ``files_changed``         | Number of files that changed                            |
++---------------------------+---------------------------------------------------------+
+| ``files_unmodified``      | Number of files that did not change                     |
++---------------------------+---------------------------------------------------------+
+| ``dirs_new``              | Number of new directories                               |
++---------------------------+---------------------------------------------------------+
+| ``dirs_changed``          | Number of directories that changed                      |
++---------------------------+---------------------------------------------------------+
+| ``dirs_unmodified``       | Number of directories that did not change               |
++---------------------------+---------------------------------------------------------+
+| ``data_blobs``            | Number of data blobs                                    |
++---------------------------+---------------------------------------------------------+
+| ``tree_blobs``            | Number of tree blobs                                    |
++---------------------------+---------------------------------------------------------+
+| ``data_added``            | Amount of (uncompressed) data added, in bytes           |
++---------------------------+---------------------------------------------------------+
+| ``data_added_packed``     | Amount of data added (after compression), in bytes      |
++---------------------------+---------------------------------------------------------+
+| ``total_files_processed`` | Total number of files processed                         |
++---------------------------+---------------------------------------------------------+
+| ``total_bytes_processed`` | Total number of bytes processed                         |
++---------------------------+---------------------------------------------------------+
+
 
 stats
 -----

go.mod

@@ -1,8 +1,8 @@
 module github.com/restic/restic
 
 require (
-	cloud.google.com/go/storage v1.39.0
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2
+	cloud.google.com/go/storage v1.40.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.1
 	github.com/Backblaze/blazer v0.6.1
@@ -17,6 +17,7 @@ require (
 	github.com/minio/minio-go/v7 v7.0.66
 	github.com/minio/sha256-simd v1.0.1
 	github.com/ncw/swift/v2 v2.0.2
+	github.com/peterbourgon/unixtransport v0.0.4
 	github.com/pkg/errors v0.9.1
 	github.com/pkg/profile v1.7.0
 	github.com/pkg/sftp v1.13.6
@@ -25,22 +26,22 @@ require (
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
 	go.uber.org/automaxprocs v1.5.3
-	golang.org/x/crypto v0.19.0
-	golang.org/x/net v0.21.0
-	golang.org/x/oauth2 v0.17.0
+	golang.org/x/crypto v0.21.0
+	golang.org/x/net v0.23.0
+	golang.org/x/oauth2 v0.18.0
 	golang.org/x/sync v0.6.0
-	golang.org/x/sys v0.17.0
-	golang.org/x/term v0.17.0
+	golang.org/x/sys v0.18.0
+	golang.org/x/term v0.18.0
 	golang.org/x/text v0.14.0
 	golang.org/x/time v0.5.0
-	google.golang.org/api v0.166.0
+	google.golang.org/api v0.170.0
 )
 
 require (
-	cloud.google.com/go v0.112.0 // indirect
+	cloud.google.com/go v0.112.1 // indirect
 	cloud.google.com/go/compute v1.24.0 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v1.1.6 // indirect
+	cloud.google.com/go/iam v1.1.7 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect
@@ -51,12 +52,12 @@ require (
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
-	github.com/googleapis/gax-go/v2 v2.12.1 // indirect
+	github.com/googleapis/gax-go/v2 v2.12.3 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
@@ -71,17 +72,17 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.48.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.48.0 // indirect
-	go.opentelemetry.io/otel v1.23.0 // indirect
-	go.opentelemetry.io/otel/metric v1.23.0 // indirect
-	go.opentelemetry.io/otel/trace v1.23.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20240221002015-b0ce06bbee7c // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9 // indirect
-	google.golang.org/grpc v1.61.1 // indirect
-	google.golang.org/protobuf v1.32.0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 // indirect
+	google.golang.org/grpc v1.62.1 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,16 +1,16 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM=
-cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4=
+cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM=
+cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4=
 cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg=
 cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40=
 cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
 cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
-cloud.google.com/go/iam v1.1.6 h1:bEa06k05IO4f4uJonbB5iAgKTPpABy1ayxaIZV/GHVc=
-cloud.google.com/go/iam v1.1.6/go.mod h1:O0zxdPeGBoFdWW3HWmBxJsk0pfvNM/p/qa82rWOGTwI=
-cloud.google.com/go/storage v1.39.0 h1:brbjUa4hbDHhpQf48tjqMaXEV+f1OGoaTmQau9tmCsA=
-cloud.google.com/go/storage v1.39.0/go.mod h1:OAEj/WZwUYjA3YHQ10/YcN9ttGuEpLwvaoyBXIPikEk=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 h1:c4k2FIYIh4xtwqrQwV0Ct1v5+ehlNXj5NI/MWVsiTkQ=
-github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2/go.mod h1:5FDJtLEO/GxwNgUxbwrY3LP0pEoThTQJtk2oysdXHxM=
+cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM=
+cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA=
+cloud.google.com/go/storage v1.40.0 h1:VEpDQV5CJxFmJ6ueWNsKxcr1QAYOXEgxDa+sBbJahPw=
+cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2ucUMY7Ir0g=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 h1:n1DH8TPV4qqPTje2RcUBYwtrTWlabVp4n46+74X2pn4=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0/go.mod h1:HDcZnuGbiyppErN6lB+idp4CKhjbc8gwjto6OPpyggM=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ=
 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo=
 github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ=
@@ -36,7 +36,6 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
-github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101 h1:7To3pQ+pZo0i3dsWEbinPNFs5gPSBOsJtx3wTT94VBY=
 github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -54,7 +53,6 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA=
 github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
 github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
@@ -84,8 +82,8 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -107,8 +105,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs=
 github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
-github.com/googleapis/gax-go/v2 v2.12.1 h1:9F8GV9r9ztXyAi00gsMQHNoF51xPZm8uj1dpYt2ZETM=
-github.com/googleapis/gax-go/v2 v2.12.1/go.mod h1:61M8vcyyXR2kqKFxKrfA22jaA8JGF7Dc8App1U3H6jc=
+github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA=
+github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
@@ -128,6 +126,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
 github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw=
@@ -141,6 +140,11 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk=
 github.com/ncw/swift/v2 v2.0.2/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg=
+github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU=
+github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/peterbourgon/ff/v3 v3.3.1/go.mod h1:zjJVUhx+twciwfDl0zBcFzl4dW8axCRyXE/eKY9RztQ=
+github.com/peterbourgon/unixtransport v0.0.4 h1:UTF0FxXCAglvoZz9jaGPYjEg52DjBLDYGMJvJni6Tfw=
+github.com/peterbourgon/unixtransport v0.0.4/go.mod h1:o8aUkOCa8W/BIXpi15uKvbSabjtBh0JhSOJGSfoOhAU=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ=
 github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -184,17 +188,17 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.48.0 h1:P+/g8GpuJGYbOp2tAdKrIPUX9JO02q8Q0YNlHolpibA=
-go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.48.0/go.mod h1:tIKj3DbO8N9Y2xo52og3irLsPI4GW02DSMtrVgNMgxg=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.48.0 h1:doUP+ExOpH3spVTLS0FcWGLnQrPct/hD/bCPbDRUEAU=
-go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.48.0/go.mod h1:rdENBZMT2OE6Ne/KLwpiXudnAsbdrdBaqBvTN8M8BgA=
-go.opentelemetry.io/otel v1.23.0 h1:Df0pqjqExIywbMCMTxkAwzjLZtRf+bBKLbUcpxO2C9E=
-go.opentelemetry.io/otel v1.23.0/go.mod h1:YCycw9ZeKhcJFrb34iVSkyT0iczq/zYDtZYFufObyB0=
-go.opentelemetry.io/otel/metric v1.23.0 h1:pazkx7ss4LFVVYSxYew7L5I6qvLXHA0Ap2pwV+9Cnpo=
-go.opentelemetry.io/otel/metric v1.23.0/go.mod h1:MqUW2X2a6Q8RN96E2/nqNoT+z9BSms20Jb7Bbp+HiTo=
-go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8=
-go.opentelemetry.io/otel/trace v1.23.0 h1:37Ik5Ib7xfYVb4V1UtnT97T1jI+AoIYkJyPkuL4iJgI=
-go.opentelemetry.io/otel/trace v1.23.0/go.mod h1:GSGTbIClEsuZrGIzoEHqsVfxgn5UkggkflQwDScNUsk=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg=
+go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
+go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
+go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
+go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI=
+go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
+go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw=
+go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
+go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
 go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8=
 go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -202,14 +206,15 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
-golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -221,16 +226,18 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ=
-golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA=
+golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI=
+golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
 golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -246,14 +253,16 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
@@ -271,12 +280,13 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200423201157-2723c5de0d66/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU=
-google.golang.org/api v0.166.0 h1:6m4NUwrZYhAaVIHZWxaKjw1L1vNAjtMwORmKRyEEo24=
-google.golang.org/api v0.166.0/go.mod h1:4FcBc686KFi7QI/U51/2GKKevfZMpM17sCdibqe/bSA=
+google.golang.org/api v0.170.0 h1:zMaruDePM88zxZBG+NG8+reALO2rfLhe/JShitLyT48=
+google.golang.org/api v0.170.0/go.mod h1:/xql9M2btF85xac/VAm4PsLMTLVGUOpq4BE9R8jyNy8=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
@@ -286,17 +296,17 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y=
 google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s=
-google.golang.org/genproto/googleapis/api v0.0.0-20240221002015-b0ce06bbee7c h1:9g7erC9qu44ks7UK4gDNlnk4kOxZG707xKm4jVniy6o=
-google.golang.org/genproto/googleapis/api v0.0.0-20240221002015-b0ce06bbee7c/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9 h1:hZB7eLIaYlW9qXRfCq/qDaPdbeY3757uARz5Vvfv+cY=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:YUWgXUFRPfoYK1IHMuxH5K6nPEXSCzIMljnQ59lLRCk=
+google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c h1:kaI7oewGK5YnVwj+Y+EJBO/YN1ht8iTL9XkFHtVZLsc=
+google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 h1:9IZDv+/GcI6u+a4jRFRLxQs0RUCfavGfoOgEW6jpkI0=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
 google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
-google.golang.org/grpc v1.61.1 h1:kLAiWrZs7YeDM6MumDe7m3y4aM6wacLzM1Y/wiLP9XY=
-google.golang.org/grpc v1.61.1/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs=
+google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk=
+google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -308,13 +318,14 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
-google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
 gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

helpers/prepare-release/main.go

@@ -303,7 +303,7 @@ func generateFiles() {
 	}
 }
 
-var versionPattern = `var version = ".*"`
+var versionPattern = `const version = ".*"`
 
 const versionCodeFile = "cmd/restic/global.go"
 
@@ -313,7 +313,7 @@ func updateVersion() {
 		die("unable to write version to file: %v", err)
 	}
 
-	newVersion := fmt.Sprintf("var version = %q", opts.Version)
+	newVersion := fmt.Sprintf("const version = %q", opts.Version)
 	replace(versionCodeFile, versionPattern, newVersion)
 
 	if len(uncommittedChanges("VERSION")) > 0 || len(uncommittedChanges(versionCodeFile)) > 0 {
@@ -323,7 +323,7 @@ func updateVersion() {
 }
 
 func updateVersionDev() {
-	newVersion := fmt.Sprintf(`var version = "%s-dev (compiled manually)"`, opts.Version)
+	newVersion := fmt.Sprintf(`const version = "%s-dev (compiled manually)"`, opts.Version)
 	replace(versionCodeFile, versionPattern, newVersion)
 
 	msg("committing cmd/restic/global.go with dev version")

internal/archiver/archiver.go

@@ -8,10 +8,12 @@ import (
 	"runtime"
 	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/feature"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
 	"golang.org/x/sync/errgroup"
@@ -40,6 +42,18 @@ type ItemStats struct {
 	TreeSizeInRepo uint64 // sum of the bytes added to the repo (including compression and crypto overhead)
 }
 
+type ChangeStats struct {
+	New       uint
+	Changed   uint
+	Unchanged uint
+}
+
+type Summary struct {
+	Files, Dirs    ChangeStats
+	ProcessedBytes uint64
+	ItemStats
+}
+
 // Add adds other to the current ItemStats.
 func (s *ItemStats) Add(other ItemStats) {
 	s.DataBlobs += other.DataBlobs
@@ -61,6 +75,8 @@ type Archiver struct {
 	blobSaver *BlobSaver
 	fileSaver *FileSaver
 	treeSaver *TreeSaver
+	mu        sync.Mutex
+	summary   *Summary
 
 	// Error is called for all errors that occur during backup.
 	Error ErrorFunc
@@ -182,12 +198,58 @@ func (arch *Archiver) error(item string, err error) error {
 	return errf
 }
 
+func (arch *Archiver) trackItem(item string, previous, current *restic.Node, s ItemStats, d time.Duration) {
+	arch.CompleteItem(item, previous, current, s, d)
+
+	arch.mu.Lock()
+	defer arch.mu.Unlock()
+
+	arch.summary.ItemStats.Add(s)
+
+	if current != nil {
+		arch.summary.ProcessedBytes += current.Size
+	} else {
+		// last item or an error occurred
+		return
+	}
+
+	switch current.Type {
+	case "dir":
+		switch {
+		case previous == nil:
+			arch.summary.Dirs.New++
+		case previous.Equals(*current):
+			arch.summary.Dirs.Unchanged++
+		default:
+			arch.summary.Dirs.Changed++
+		}
+
+	case "file":
+		switch {
+		case previous == nil:
+			arch.summary.Files.New++
+		case previous.Equals(*current):
+			arch.summary.Files.Unchanged++
+		default:
+			arch.summary.Files.Changed++
+		}
+	}
+}
+
 // nodeFromFileInfo returns the restic node from an os.FileInfo.
-func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo) (*restic.Node, error) {
-	node, err := restic.NodeFromFileInfo(filename, fi)
+func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) {
+	node, err := restic.NodeFromFileInfo(filename, fi, ignoreXattrListError)
 	if !arch.WithAtime {
 		node.AccessTime = node.ModTime
 	}
+	if feature.Flag.Enabled(feature.DeviceIDForHardlinks) {
+		if node.Links == 1 || node.Type == "dir" {
+			// the DeviceID is only necessary for hardlinked files
+			// when using subvolumes or snapshots their deviceIDs tend to change which causes
+			// restic to upload new tree blobs
+			node.DeviceID = 0
+		}
+	}
 	// overwrite name to match that within the snapshot
 	node.Name = path.Base(snPath)
 	if err != nil {
@@ -222,12 +284,12 @@ func (arch *Archiver) wrapLoadTreeError(id restic.ID, err error) error {
 	return err
 }
 
-// SaveDir stores a directory in the repo and returns the node. snPath is the
+// saveDir stores a directory in the repo and returns the node. snPath is the
 // path within the current snapshot.
-func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete CompleteFunc) (d FutureNode, err error) {
+func (arch *Archiver) saveDir(ctx context.Context, snPath string, dir string, fi os.FileInfo, previous *restic.Tree, complete CompleteFunc) (d FutureNode, err error) {
 	debug.Log("%v %v", snPath, dir)
 
-	treeNode, err := arch.nodeFromFileInfo(snPath, dir, fi)
+	treeNode, err := arch.nodeFromFileInfo(snPath, dir, fi, false)
 	if err != nil {
 		return FutureNode{}, err
 	}
@@ -250,7 +312,7 @@ func (arch *Archiver) SaveDir(ctx context.Context, snPath string, dir string, fi
 		pathname := arch.FS.Join(dir, name)
 		oldNode := previous.Find(name)
 		snItem := join(snPath, name)
-		fn, excluded, err := arch.Save(ctx, snItem, pathname, oldNode)
+		fn, excluded, err := arch.save(ctx, snItem, pathname, oldNode)
 
 		// return error early if possible
 		if err != nil {
@@ -318,6 +380,7 @@ func (fn *FutureNode) take(ctx context.Context) futureNodeResult {
 			return res
 		}
 	case <-ctx.Done():
+		return futureNodeResult{err: ctx.Err()}
 	}
 	return futureNodeResult{err: errors.Errorf("no result")}
 }
@@ -334,14 +397,14 @@ func (arch *Archiver) allBlobsPresent(previous *restic.Node) bool {
 	return true
 }
 
-// Save saves a target (file or directory) to the repo. If the item is
+// save saves a target (file or directory) to the repo. If the item is
 // excluded, this function returns a nil node and error, with excluded set to
 // true.
 //
 // Errors and completion needs to be handled by the caller.
 //
 // snPath is the path within the current snapshot.
-func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous *restic.Node) (fn FutureNode, excluded bool, err error) {
+func (arch *Archiver) save(ctx context.Context, snPath, target string, previous *restic.Node) (fn FutureNode, excluded bool, err error) {
 	start := time.Now()
 
 	debug.Log("%v target %q, previous %v", snPath, target, previous)
@@ -380,9 +443,9 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
 		if previous != nil && !fileChanged(fi, previous, arch.ChangeIgnoreFlags) {
 			if arch.allBlobsPresent(previous) {
 				debug.Log("%v hasn't changed, using old list of blobs", target)
-				arch.CompleteItem(snPath, previous, previous, ItemStats{}, time.Since(start))
+				arch.trackItem(snPath, previous, previous, ItemStats{}, time.Since(start))
 				arch.CompleteBlob(previous.Size)
-				node, err := arch.nodeFromFileInfo(snPath, target, fi)
+				node, err := arch.nodeFromFileInfo(snPath, target, fi, false)
 				if err != nil {
 					return FutureNode{}, false, err
 				}
@@ -445,9 +508,9 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
 		fn = arch.fileSaver.Save(ctx, snPath, target, file, fi, func() {
 			arch.StartFile(snPath)
 		}, func() {
-			arch.CompleteItem(snPath, nil, nil, ItemStats{}, 0)
+			arch.trackItem(snPath, nil, nil, ItemStats{}, 0)
 		}, func(node *restic.Node, stats ItemStats) {
-			arch.CompleteItem(snPath, previous, node, stats, time.Since(start))
+			arch.trackItem(snPath, previous, node, stats, time.Since(start))
 		})
 
 	case fi.IsDir():
@@ -462,9 +525,9 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
 			return FutureNode{}, false, err
 		}
 
-		fn, err = arch.SaveDir(ctx, snPath, target, fi, oldSubtree,
+		fn, err = arch.saveDir(ctx, snPath, target, fi, oldSubtree,
 			func(node *restic.Node, stats ItemStats) {
-				arch.CompleteItem(snItem, previous, node, stats, time.Since(start))
+				arch.trackItem(snItem, previous, node, stats, time.Since(start))
 			})
 		if err != nil {
 			debug.Log("SaveDir for %v returned error: %v", snPath, err)
@@ -478,7 +541,7 @@ func (arch *Archiver) Save(ctx context.Context, snPath, target string, previous
 	default:
 		debug.Log("  %v other", target)
 
-		node, err := arch.nodeFromFileInfo(snPath, target, fi)
+		node, err := arch.nodeFromFileInfo(snPath, target, fi, false)
 		if err != nil {
 			return FutureNode{}, false, err
 		}
@@ -545,9 +608,9 @@ func (arch *Archiver) statDir(dir string) (os.FileInfo, error) {
 	return fi, nil
 }
 
-// SaveTree stores a Tree in the repo, returned is the tree. snPath is the path
+// saveTree stores a Tree in the repo, returned is the tree. snPath is the path
 // within the current snapshot.
-func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree, previous *restic.Tree, complete CompleteFunc) (FutureNode, int, error) {
+func (arch *Archiver) saveTree(ctx context.Context, snPath string, atree *Tree, previous *restic.Tree, complete CompleteFunc) (FutureNode, int, error) {
 
 	var node *restic.Node
 	if snPath != "/" {
@@ -561,7 +624,9 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree,
 		}
 
 		debug.Log("%v, dir node data loaded from %v", snPath, atree.FileInfoPath)
-		node, err = arch.nodeFromFileInfo(snPath, atree.FileInfoPath, fi)
+		// in some cases reading xattrs for directories above the backup target is not allowed
+		// thus ignore errors for such folders.
+		node, err = arch.nodeFromFileInfo(snPath, atree.FileInfoPath, fi, true)
 		if err != nil {
 			return FutureNode{}, 0, err
 		}
@@ -585,7 +650,7 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree,
 
 		// this is a leaf node
 		if subatree.Leaf() {
-			fn, excluded, err := arch.Save(ctx, join(snPath, name), subatree.Path, previous.Find(name))
+			fn, excluded, err := arch.save(ctx, join(snPath, name), subatree.Path, previous.Find(name))
 
 			if err != nil {
 				err = arch.error(subatree.Path, err)
@@ -619,8 +684,8 @@ func (arch *Archiver) SaveTree(ctx context.Context, snPath string, atree *Tree,
 		}
 
 		// not a leaf node, archive subtree
-		fn, _, err := arch.SaveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *restic.Node, is ItemStats) {
-			arch.CompleteItem(snItem, oldNode, n, is, time.Since(start))
+		fn, _, err := arch.saveTree(ctx, join(snPath, name), &subatree, oldSubtree, func(n *restic.Node, is ItemStats) {
+			arch.trackItem(snItem, oldNode, n, is, time.Since(start))
 		})
 		if err != nil {
 			return FutureNode{}, 0, err
@@ -688,6 +753,7 @@ type SnapshotOptions struct {
 	Tags           restic.TagList
 	Hostname       string
 	Excludes       []string
+	BackupStart    time.Time
 	Time           time.Time
 	ParentSnapshot *restic.Snapshot
 	ProgramVersion string
@@ -738,15 +804,17 @@ func (arch *Archiver) stopWorkers() {
 }
 
 // Snapshot saves several targets and returns a snapshot.
-func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts SnapshotOptions) (*restic.Snapshot, restic.ID, error) {
+func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts SnapshotOptions) (*restic.Snapshot, restic.ID, *Summary, error) {
+	arch.summary = &Summary{}
+
 	cleanTargets, err := resolveRelativeTargets(arch.FS, targets)
 	if err != nil {
-		return nil, restic.ID{}, err
+		return nil, restic.ID{}, nil, err
 	}
 
 	atree, err := NewTree(arch.FS, cleanTargets)
 	if err != nil {
-		return nil, restic.ID{}, err
+		return nil, restic.ID{}, nil, err
 	}
 
 	var rootTreeID restic.ID
@@ -762,8 +830,8 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
 			arch.runWorkers(wgCtx, wg)
 
 			debug.Log("starting snapshot")
-			fn, nodeCount, err := arch.SaveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(_ *restic.Node, is ItemStats) {
-				arch.CompleteItem("/", nil, nil, is, time.Since(start))
+			fn, nodeCount, err := arch.saveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(_ *restic.Node, is ItemStats) {
+				arch.trackItem("/", nil, nil, is, time.Since(start))
 			})
 			if err != nil {
 				return err
@@ -799,12 +867,12 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
 	})
 	err = wgUp.Wait()
 	if err != nil {
-		return nil, restic.ID{}, err
+		return nil, restic.ID{}, nil, err
 	}
 
 	sn, err := restic.NewSnapshot(targets, opts.Tags, opts.Hostname, opts.Time)
 	if err != nil {
-		return nil, restic.ID{}, err
+		return nil, restic.ID{}, nil, err
 	}
 
 	sn.ProgramVersion = opts.ProgramVersion
@@ -813,11 +881,28 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps
 		sn.Parent = opts.ParentSnapshot.ID()
 	}
 	sn.Tree = &rootTreeID
+	sn.Summary = &restic.SnapshotSummary{
+		BackupStart: opts.BackupStart,
+		BackupEnd:   time.Now(),
+
+		FilesNew:            arch.summary.Files.New,
+		FilesChanged:        arch.summary.Files.Changed,
+		FilesUnmodified:     arch.summary.Files.Unchanged,
+		DirsNew:             arch.summary.Dirs.New,
+		DirsChanged:         arch.summary.Dirs.Changed,
+		DirsUnmodified:      arch.summary.Dirs.Unchanged,
+		DataBlobs:           arch.summary.ItemStats.DataBlobs,
+		TreeBlobs:           arch.summary.ItemStats.TreeBlobs,
+		DataAdded:           arch.summary.ItemStats.DataSize + arch.summary.ItemStats.TreeSize,
+		DataAddedPacked:     arch.summary.ItemStats.DataSizeInRepo + arch.summary.ItemStats.TreeSizeInRepo,
+		TotalFilesProcessed: arch.summary.Files.New + arch.summary.Files.Changed + arch.summary.Files.Unchanged,
+		TotalBytesProcessed: arch.summary.ProcessedBytes,
+	}
 
 	id, err := restic.SaveSnapshot(ctx, arch.Repo, sn)
 	if err != nil {
-		return nil, restic.ID{}, err
+		return nil, restic.ID{}, nil, err
 	}
 
-	return sn, id, nil
+	return sn, id, arch.summary, nil
 }

internal/archiver/archiver_test.go

@@ -19,15 +19,16 @@ import (
 	"github.com/restic/restic/internal/backend/mem"
 	"github.com/restic/restic/internal/checker"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/feature"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
-	restictest "github.com/restic/restic/internal/test"
+	rtest "github.com/restic/restic/internal/test"
 	"golang.org/x/sync/errgroup"
 )
 
 func prepareTempdirRepoSrc(t testing.TB, src TestDir) (string, restic.Repository) {
-	tempdir := restictest.TempDir(t)
+	tempdir := rtest.TempDir(t)
 	repo := repository.TestRepository(t)
 
 	TestCreateFiles(t, tempdir, src)
@@ -132,7 +133,7 @@ func TestArchiverSaveFile(t *testing.T) {
 	var tests = []TestFile{
 		{Content: ""},
 		{Content: "foo"},
-		{Content: string(restictest.Random(23, 12*1024*1024+1287898))},
+		{Content: string(rtest.Random(23, 12*1024*1024+1287898))},
 	}
 
 	for _, testfile := range tests {
@@ -165,7 +166,7 @@ func TestArchiverSaveFileReaderFS(t *testing.T) {
 		Data string
 	}{
 		{Data: "foo"},
-		{Data: string(restictest.Random(23, 12*1024*1024+1287898))},
+		{Data: string(rtest.Random(23, 12*1024*1024+1287898))},
 	}
 
 	for _, test := range tests {
@@ -207,7 +208,7 @@ func TestArchiverSave(t *testing.T) {
 	var tests = []TestFile{
 		{Content: ""},
 		{Content: "foo"},
-		{Content: string(restictest.Random(23, 12*1024*1024+1287898))},
+		{Content: string(rtest.Random(23, 12*1024*1024+1287898))},
 	}
 
 	for _, testfile := range tests {
@@ -226,8 +227,9 @@ func TestArchiverSave(t *testing.T) {
 				return err
 			}
 			arch.runWorkers(ctx, wg)
+			arch.summary = &Summary{}
 
-			node, excluded, err := arch.Save(ctx, "/", filepath.Join(tempdir, "file"), nil)
+			node, excluded, err := arch.save(ctx, "/", filepath.Join(tempdir, "file"), nil)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -275,7 +277,7 @@ func TestArchiverSaveReaderFS(t *testing.T) {
 		Data string
 	}{
 		{Data: "foo"},
-		{Data: string(restictest.Random(23, 12*1024*1024+1287898))},
+		{Data: string(rtest.Random(23, 12*1024*1024+1287898))},
 	}
 
 	for _, test := range tests {
@@ -303,8 +305,9 @@ func TestArchiverSaveReaderFS(t *testing.T) {
 				return err
 			}
 			arch.runWorkers(ctx, wg)
+			arch.summary = &Summary{}
 
-			node, excluded, err := arch.Save(ctx, "/", filename, nil)
+			node, excluded, err := arch.save(ctx, "/", filename, nil)
 			t.Logf("Save returned %v %v", node, err)
 			if err != nil {
 				t.Fatal(err)
@@ -351,7 +354,7 @@ func TestArchiverSaveReaderFS(t *testing.T) {
 func BenchmarkArchiverSaveFileSmall(b *testing.B) {
 	const fileSize = 4 * 1024
 	d := TestDir{"file": TestFile{
-		Content: string(restictest.Random(23, fileSize)),
+		Content: string(rtest.Random(23, fileSize)),
 	}}
 
 	b.SetBytes(fileSize)
@@ -383,7 +386,7 @@ func BenchmarkArchiverSaveFileSmall(b *testing.B) {
 func BenchmarkArchiverSaveFileLarge(b *testing.B) {
 	const fileSize = 40*1024*1024 + 1287898
 	d := TestDir{"file": TestFile{
-		Content: string(restictest.Random(23, fileSize)),
+		Content: string(rtest.Random(23, fileSize)),
 	}}
 
 	b.SetBytes(fileSize)
@@ -459,14 +462,14 @@ func appendToFile(t testing.TB, filename string, data []byte) {
 }
 
 func TestArchiverSaveFileIncremental(t *testing.T) {
-	tempdir := restictest.TempDir(t)
+	tempdir := rtest.TempDir(t)
 
 	repo := &blobCountingRepo{
 		Repository: repository.TestRepository(t),
 		saved:      make(map[restic.BlobHandle]uint),
 	}
 
-	data := restictest.Random(23, 512*1024+887898)
+	data := rtest.Random(23, 512*1024+887898)
 	testfile := filepath.Join(tempdir, "testfile")
 
 	for i := 0; i < 3; i++ {
@@ -509,12 +512,12 @@ func chmodTwice(t testing.TB, name string) {
 	// POSIX says that ctime is updated "even if the file status does not
 	// change", but let's make sure it does change, just in case.
 	err := os.Chmod(name, 0700)
-	restictest.OK(t, err)
+	rtest.OK(t, err)
 
 	sleep()
 
 	err = os.Chmod(name, 0600)
-	restictest.OK(t, err)
+	rtest.OK(t, err)
 }
 
 func lstat(t testing.TB, name string) os.FileInfo {
@@ -553,7 +556,7 @@ func rename(t testing.TB, oldname, newname string) {
 }
 
 func nodeFromFI(t testing.TB, filename string, fi os.FileInfo) *restic.Node {
-	node, err := restic.NodeFromFileInfo(filename, fi)
+	node, err := restic.NodeFromFileInfo(filename, fi, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -673,7 +676,7 @@ func TestFileChanged(t *testing.T) {
 				t.Skip("don't run test on Windows")
 			}
 
-			tempdir := restictest.TempDir(t)
+			tempdir := rtest.TempDir(t)
 
 			filename := filepath.Join(tempdir, "file")
 			content := defaultContent
@@ -709,7 +712,7 @@ func TestFileChanged(t *testing.T) {
 }
 
 func TestFilChangedSpecialCases(t *testing.T) {
-	tempdir := restictest.TempDir(t)
+	tempdir := rtest.TempDir(t)
 
 	filename := filepath.Join(tempdir, "file")
 	content := []byte("foobar")
@@ -743,12 +746,12 @@ func TestArchiverSaveDir(t *testing.T) {
 	}{
 		{
 			src: TestDir{
-				"targetfile": TestFile{Content: string(restictest.Random(888, 2*1024*1024+5000))},
+				"targetfile": TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))},
 			},
 			target: ".",
 			want: TestDir{
 				"targetdir": TestDir{
-					"targetfile": TestFile{Content: string(restictest.Random(888, 2*1024*1024+5000))},
+					"targetfile": TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))},
 				},
 			},
 		},
@@ -758,8 +761,8 @@ func TestArchiverSaveDir(t *testing.T) {
 					"foo":        TestFile{Content: "foo"},
 					"emptyfile":  TestFile{Content: ""},
 					"bar":        TestFile{Content: "XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"},
-					"largefile":  TestFile{Content: string(restictest.Random(888, 2*1024*1024+5000))},
-					"largerfile": TestFile{Content: string(restictest.Random(234, 5*1024*1024+5000))},
+					"largefile":  TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))},
+					"largerfile": TestFile{Content: string(rtest.Random(234, 5*1024*1024+5000))},
 				},
 			},
 			target: "targetdir",
@@ -831,13 +834,14 @@ func TestArchiverSaveDir(t *testing.T) {
 
 			arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
 			arch.runWorkers(ctx, wg)
+			arch.summary = &Summary{}
 
 			chdir := tempdir
 			if test.chdir != "" {
 				chdir = filepath.Join(chdir, test.chdir)
 			}
 
-			back := restictest.Chdir(t, chdir)
+			back := rtest.Chdir(t, chdir)
 			defer back()
 
 			fi, err := fs.Lstat(test.target)
@@ -845,7 +849,7 @@ func TestArchiverSaveDir(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			ft, err := arch.SaveDir(ctx, "/", test.target, fi, nil, nil)
+			ft, err := arch.saveDir(ctx, "/", test.target, fi, nil, nil)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -895,7 +899,7 @@ func TestArchiverSaveDir(t *testing.T) {
 }
 
 func TestArchiverSaveDirIncremental(t *testing.T) {
-	tempdir := restictest.TempDir(t)
+	tempdir := rtest.TempDir(t)
 
 	repo := &blobCountingRepo{
 		Repository: repository.TestRepository(t),
@@ -912,13 +916,14 @@ func TestArchiverSaveDirIncremental(t *testing.T) {
 
 		arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
 		arch.runWorkers(ctx, wg)
+		arch.summary = &Summary{}
 
 		fi, err := fs.Lstat(tempdir)
 		if err != nil {
 			t.Fatal(err)
 		}
 
-		ft, err := arch.SaveDir(ctx, "/", tempdir, fi, nil, nil)
+		ft, err := arch.saveDir(ctx, "/", tempdir, fi, nil, nil)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -982,9 +987,9 @@ func TestArchiverSaveDirIncremental(t *testing.T) {
 
 // bothZeroOrNeither fails the test if only one of exp, act is zero.
 func bothZeroOrNeither(tb testing.TB, exp, act uint64) {
+	tb.Helper()
 	if (exp == 0 && act != 0) || (exp != 0 && act == 0) {
-		_, file, line, _ := runtime.Caller(1)
-		tb.Fatalf("\033[31m%s:%d:\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", filepath.Base(file), line, exp, act)
+		rtest.Equals(tb, exp, act)
 	}
 }
 
@@ -1004,7 +1009,7 @@ func TestArchiverSaveTree(t *testing.T) {
 		prepare func(t testing.TB)
 		targets []string
 		want    TestDir
-		stat    ItemStats
+		stat    Summary
 	}{
 		{
 			src: TestDir{
@@ -1014,7 +1019,12 @@ func TestArchiverSaveTree(t *testing.T) {
 			want: TestDir{
 				"targetfile": TestFile{Content: string("foobar")},
 			},
-			stat: ItemStats{1, 6, 32 + 6, 0, 0, 0},
+			stat: Summary{
+				ItemStats:      ItemStats{1, 6, 32 + 6, 0, 0, 0},
+				ProcessedBytes: 6,
+				Files:          ChangeStats{1, 0, 0},
+				Dirs:           ChangeStats{0, 0, 0},
+			},
 		},
 		{
 			src: TestDir{
@@ -1026,7 +1036,12 @@ func TestArchiverSaveTree(t *testing.T) {
 				"targetfile":  TestFile{Content: string("foobar")},
 				"filesymlink": TestSymlink{Target: "targetfile"},
 			},
-			stat: ItemStats{1, 6, 32 + 6, 0, 0, 0},
+			stat: Summary{
+				ItemStats:      ItemStats{1, 6, 32 + 6, 0, 0, 0},
+				ProcessedBytes: 6,
+				Files:          ChangeStats{1, 0, 0},
+				Dirs:           ChangeStats{0, 0, 0},
+			},
 		},
 		{
 			src: TestDir{
@@ -1046,7 +1061,12 @@ func TestArchiverSaveTree(t *testing.T) {
 					"symlink": TestSymlink{Target: "subdir"},
 				},
 			},
-			stat: ItemStats{0, 0, 0, 1, 0x154, 0x16a},
+			stat: Summary{
+				ItemStats:      ItemStats{0, 0, 0, 1, 0x154, 0x16a},
+				ProcessedBytes: 0,
+				Files:          ChangeStats{0, 0, 0},
+				Dirs:           ChangeStats{1, 0, 0},
+			},
 		},
 		{
 			src: TestDir{
@@ -1070,7 +1090,12 @@ func TestArchiverSaveTree(t *testing.T) {
 					},
 				},
 			},
-			stat: ItemStats{1, 6, 32 + 6, 3, 0x47f, 0x4c1},
+			stat: Summary{
+				ItemStats:      ItemStats{1, 6, 32 + 6, 3, 0x47f, 0x4c1},
+				ProcessedBytes: 6,
+				Files:          ChangeStats{1, 0, 0},
+				Dirs:           ChangeStats{3, 0, 0},
+			},
 		},
 	}
 
@@ -1082,20 +1107,13 @@ func TestArchiverSaveTree(t *testing.T) {
 
 			arch := New(repo, testFS, Options{})
 
-			var stat ItemStats
-			lock := &sync.Mutex{}
-			arch.CompleteItem = func(item string, previous, current *restic.Node, s ItemStats, d time.Duration) {
-				lock.Lock()
-				defer lock.Unlock()
-				stat.Add(s)
-			}
-
 			wg, ctx := errgroup.WithContext(context.TODO())
 			repo.StartPackUploader(ctx, wg)
 
 			arch.runWorkers(ctx, wg)
+			arch.summary = &Summary{}
 
-			back := restictest.Chdir(t, tempdir)
+			back := rtest.Chdir(t, tempdir)
 			defer back()
 
 			if test.prepare != nil {
@@ -1107,7 +1125,7 @@ func TestArchiverSaveTree(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			fn, _, err := arch.SaveTree(ctx, "/", atree, nil, nil)
+			fn, _, err := arch.saveTree(ctx, "/", atree, nil, nil)
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -1134,11 +1152,15 @@ func TestArchiverSaveTree(t *testing.T) {
 				want = test.src
 			}
 			TestEnsureTree(context.TODO(), t, "/", repo, treeID, want)
+			stat := arch.summary
 			bothZeroOrNeither(t, uint64(test.stat.DataBlobs), uint64(stat.DataBlobs))
 			bothZeroOrNeither(t, uint64(test.stat.TreeBlobs), uint64(stat.TreeBlobs))
 			bothZeroOrNeither(t, test.stat.DataSize, stat.DataSize)
 			bothZeroOrNeither(t, test.stat.DataSizeInRepo, stat.DataSizeInRepo)
 			bothZeroOrNeither(t, test.stat.TreeSizeInRepo, stat.TreeSizeInRepo)
+			rtest.Equals(t, test.stat.ProcessedBytes, stat.ProcessedBytes)
+			rtest.Equals(t, test.stat.Files, stat.Files)
+			rtest.Equals(t, test.stat.Dirs, stat.Dirs)
 		})
 	}
 }
@@ -1386,7 +1408,7 @@ func TestArchiverSnapshot(t *testing.T) {
 				chdir = filepath.Join(chdir, filepath.FromSlash(test.chdir))
 			}
 
-			back := restictest.Chdir(t, chdir)
+			back := rtest.Chdir(t, chdir)
 			defer back()
 
 			var targets []string
@@ -1395,7 +1417,7 @@ func TestArchiverSnapshot(t *testing.T) {
 			}
 
 			t.Logf("targets: %v", targets)
-			sn, snapshotID, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()})
+			sn, snapshotID, _, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()})
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -1408,7 +1430,7 @@ func TestArchiverSnapshot(t *testing.T) {
 			}
 			TestEnsureSnapshot(t, repo, snapshotID, want)
 
-			checker.TestCheckRepo(t, repo)
+			checker.TestCheckRepo(t, repo, false)
 
 			// check that the snapshot contains the targets with absolute paths
 			for i, target := range sn.Paths {
@@ -1539,11 +1561,11 @@ func TestArchiverSnapshotSelect(t *testing.T) {
 			arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
 			arch.Select = test.selFn
 
-			back := restictest.Chdir(t, tempdir)
+			back := rtest.Chdir(t, tempdir)
 			defer back()
 
 			targets := []string{"."}
-			_, snapshotID, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()})
+			_, snapshotID, _, err := arch.Snapshot(ctx, targets, SnapshotOptions{Time: time.Now()})
 			if test.err != "" {
 				if err == nil {
 					t.Fatalf("expected error not found, got %v, wanted %q", err, test.err)
@@ -1568,7 +1590,7 @@ func TestArchiverSnapshotSelect(t *testing.T) {
 			}
 			TestEnsureSnapshot(t, repo, snapshotID, want)
 
-			checker.TestCheckRepo(t, repo)
+			checker.TestCheckRepo(t, repo, false)
 		})
 	}
 }
@@ -1616,17 +1638,85 @@ func (f MockFile) Read(p []byte) (int, error) {
 	return n, err
 }
 
+func checkSnapshotStats(t *testing.T, sn *restic.Snapshot, stat Summary) {
+	rtest.Equals(t, stat.Files.New, sn.Summary.FilesNew)
+	rtest.Equals(t, stat.Files.Changed, sn.Summary.FilesChanged)
+	rtest.Equals(t, stat.Files.Unchanged, sn.Summary.FilesUnmodified)
+	rtest.Equals(t, stat.Dirs.New, sn.Summary.DirsNew)
+	rtest.Equals(t, stat.Dirs.Changed, sn.Summary.DirsChanged)
+	rtest.Equals(t, stat.Dirs.Unchanged, sn.Summary.DirsUnmodified)
+	rtest.Equals(t, stat.ProcessedBytes, sn.Summary.TotalBytesProcessed)
+	rtest.Equals(t, stat.Files.New+stat.Files.Changed+stat.Files.Unchanged, sn.Summary.TotalFilesProcessed)
+	bothZeroOrNeither(t, uint64(stat.DataBlobs), uint64(sn.Summary.DataBlobs))
+	bothZeroOrNeither(t, uint64(stat.TreeBlobs), uint64(sn.Summary.TreeBlobs))
+	bothZeroOrNeither(t, uint64(stat.DataSize+stat.TreeSize), uint64(sn.Summary.DataAdded))
+	bothZeroOrNeither(t, uint64(stat.DataSizeInRepo+stat.TreeSizeInRepo), uint64(sn.Summary.DataAddedPacked))
+}
+
 func TestArchiverParent(t *testing.T) {
 	var tests = []struct {
-		src  TestDir
-		read map[string]int // tracks number of times a file must have been read
+		src         TestDir
+		modify      func(path string)
+		statInitial Summary
+		statSecond  Summary
 	}{
 		{
 			src: TestDir{
-				"targetfile": TestFile{Content: string(restictest.Random(888, 2*1024*1024+5000))},
+				"targetfile": TestFile{Content: string(rtest.Random(888, 2*1024*1024+5000))},
 			},
-			read: map[string]int{
-				"targetfile": 1,
+			statInitial: Summary{
+				Files:          ChangeStats{1, 0, 0},
+				Dirs:           ChangeStats{0, 0, 0},
+				ProcessedBytes: 2102152,
+				ItemStats:      ItemStats{3, 0x201593, 0x201632, 1, 0, 0},
+			},
+			statSecond: Summary{
+				Files:          ChangeStats{0, 0, 1},
+				Dirs:           ChangeStats{0, 0, 0},
+				ProcessedBytes: 2102152,
+			},
+		},
+		{
+			src: TestDir{
+				"targetDir": TestDir{
+					"targetfile":  TestFile{Content: string(rtest.Random(888, 1234))},
+					"targetfile2": TestFile{Content: string(rtest.Random(888, 1235))},
+				},
+			},
+			statInitial: Summary{
+				Files:          ChangeStats{2, 0, 0},
+				Dirs:           ChangeStats{1, 0, 0},
+				ProcessedBytes: 2469,
+				ItemStats:      ItemStats{2, 0xe1c, 0xcd9, 2, 0, 0},
+			},
+			statSecond: Summary{
+				Files:          ChangeStats{0, 0, 2},
+				Dirs:           ChangeStats{0, 0, 1},
+				ProcessedBytes: 2469,
+			},
+		},
+		{
+			src: TestDir{
+				"targetDir": TestDir{
+					"targetfile": TestFile{Content: string(rtest.Random(888, 1234))},
+				},
+				"targetfile2": TestFile{Content: string(rtest.Random(888, 1235))},
+			},
+			modify: func(path string) {
+				remove(t, filepath.Join(path, "targetDir", "targetfile"))
+				save(t, filepath.Join(path, "targetfile2"), []byte("foobar"))
+			},
+			statInitial: Summary{
+				Files:          ChangeStats{2, 0, 0},
+				Dirs:           ChangeStats{1, 0, 0},
+				ProcessedBytes: 2469,
+				ItemStats:      ItemStats{2, 0xe13, 0xcf8, 2, 0, 0},
+			},
+			statSecond: Summary{
+				Files:          ChangeStats{0, 1, 0},
+				Dirs:           ChangeStats{0, 1, 0},
+				ProcessedBytes: 6,
+				ItemStats:      ItemStats{1, 0x305, 0x233, 2, 0, 0},
 			},
 		},
 	}
@@ -1645,10 +1735,10 @@ func TestArchiverParent(t *testing.T) {
 
 			arch := New(repo, testFS, Options{})
 
-			back := restictest.Chdir(t, tempdir)
+			back := rtest.Chdir(t, tempdir)
 			defer back()
 
-			firstSnapshot, firstSnapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
+			firstSnapshot, firstSnapshotID, summary, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -1673,38 +1763,38 @@ func TestArchiverParent(t *testing.T) {
 				}
 				return nil
 			})
+			rtest.Equals(t, test.statInitial.Files, summary.Files)
+			rtest.Equals(t, test.statInitial.Dirs, summary.Dirs)
+			rtest.Equals(t, test.statInitial.ProcessedBytes, summary.ProcessedBytes)
+			checkSnapshotStats(t, firstSnapshot, test.statInitial)
+
+			if test.modify != nil {
+				test.modify(tempdir)
+			}
 
 			opts := SnapshotOptions{
 				Time:           time.Now(),
 				ParentSnapshot: firstSnapshot,
 			}
-			_, secondSnapshotID, err := arch.Snapshot(ctx, []string{"."}, opts)
+			testFS.bytesRead = map[string]int{}
+			secondSnapshot, secondSnapshotID, summary, err := arch.Snapshot(ctx, []string{"."}, opts)
 			if err != nil {
 				t.Fatal(err)
 			}
 
-			// check that all files still been read exactly once
-			TestWalkFiles(t, ".", test.src, func(filename string, item interface{}) error {
-				file, ok := item.(TestFile)
-				if !ok {
-					return nil
-				}
-
-				n, ok := testFS.bytesRead[filename]
-				if !ok {
-					t.Fatalf("file %v was not read at all", filename)
-				}
-
-				if n != len(file.Content) {
-					t.Fatalf("file %v: read %v bytes, wanted %v bytes", filename, n, len(file.Content))
-				}
-				return nil
-			})
+			if test.modify == nil {
+				// check that no files were read this time
+				rtest.Equals(t, map[string]int{}, testFS.bytesRead)
+			}
+			rtest.Equals(t, test.statSecond.Files, summary.Files)
+			rtest.Equals(t, test.statSecond.Dirs, summary.Dirs)
+			rtest.Equals(t, test.statSecond.ProcessedBytes, summary.ProcessedBytes)
+			checkSnapshotStats(t, secondSnapshot, test.statSecond)
 
 			t.Logf("second backup saved as %v", secondSnapshotID.Str())
 			t.Logf("testfs: %v", testFS)
 
-			checker.TestCheckRepo(t, repo)
+			checker.TestCheckRepo(t, repo, false)
 		})
 	}
 }
@@ -1804,7 +1894,7 @@ func TestArchiverErrorReporting(t *testing.T) {
 
 			tempdir, repo := prepareTempdirRepoSrc(t, test.src)
 
-			back := restictest.Chdir(t, tempdir)
+			back := rtest.Chdir(t, tempdir)
 			defer back()
 
 			if test.prepare != nil {
@@ -1814,7 +1904,7 @@ func TestArchiverErrorReporting(t *testing.T) {
 			arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
 			arch.Error = test.errFn
 
-			_, snapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
+			_, snapshotID, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
 			if test.mustError {
 				if err != nil {
 					t.Logf("found expected error (%v), skipping further checks", err)
@@ -1837,7 +1927,7 @@ func TestArchiverErrorReporting(t *testing.T) {
 			}
 			TestEnsureSnapshot(t, repo, snapshotID, want)
 
-			checker.TestCheckRepo(t, repo)
+			checker.TestCheckRepo(t, repo, false)
 		})
 	}
 }
@@ -1874,7 +1964,7 @@ func TestArchiverContextCanceled(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
 
-	tempdir := restictest.TempDir(t)
+	tempdir := rtest.TempDir(t)
 	TestCreateFiles(t, tempdir, TestDir{
 		"targetfile": TestFile{Content: "foobar"},
 	})
@@ -1882,12 +1972,12 @@ func TestArchiverContextCanceled(t *testing.T) {
 	// Ensure that the archiver itself reports the canceled context and not just the backend
 	repo := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0, repository.Options{})
 
-	back := restictest.Chdir(t, tempdir)
+	back := rtest.Chdir(t, tempdir)
 	defer back()
 
 	arch := New(repo, fs.Track{FS: fs.Local{}}, Options{})
 
-	_, snapshotID, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
+	_, snapshotID, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
 
 	if err != nil {
 		t.Logf("found expected error (%v)", err)
@@ -1968,16 +2058,16 @@ func TestArchiverAbortEarlyOnError(t *testing.T) {
 		{
 			src: TestDir{
 				"dir": TestDir{
-					"file0": TestFile{Content: string(restictest.Random(0, 1024))},
-					"file1": TestFile{Content: string(restictest.Random(1, 1024))},
-					"file2": TestFile{Content: string(restictest.Random(2, 1024))},
-					"file3": TestFile{Content: string(restictest.Random(3, 1024))},
-					"file4": TestFile{Content: string(restictest.Random(4, 1024))},
-					"file5": TestFile{Content: string(restictest.Random(5, 1024))},
-					"file6": TestFile{Content: string(restictest.Random(6, 1024))},
-					"file7": TestFile{Content: string(restictest.Random(7, 1024))},
-					"file8": TestFile{Content: string(restictest.Random(8, 1024))},
-					"file9": TestFile{Content: string(restictest.Random(9, 1024))},
+					"file0": TestFile{Content: string(rtest.Random(0, 1024))},
+					"file1": TestFile{Content: string(rtest.Random(1, 1024))},
+					"file2": TestFile{Content: string(rtest.Random(2, 1024))},
+					"file3": TestFile{Content: string(rtest.Random(3, 1024))},
+					"file4": TestFile{Content: string(rtest.Random(4, 1024))},
+					"file5": TestFile{Content: string(rtest.Random(5, 1024))},
+					"file6": TestFile{Content: string(rtest.Random(6, 1024))},
+					"file7": TestFile{Content: string(rtest.Random(7, 1024))},
+					"file8": TestFile{Content: string(rtest.Random(8, 1024))},
+					"file9": TestFile{Content: string(rtest.Random(9, 1024))},
 				},
 			},
 			wantOpen: map[string]uint{
@@ -2002,7 +2092,7 @@ func TestArchiverAbortEarlyOnError(t *testing.T) {
 
 			tempdir, repo := prepareTempdirRepoSrc(t, test.src)
 
-			back := restictest.Chdir(t, tempdir)
+			back := rtest.Chdir(t, tempdir)
 			defer back()
 
 			testFS := &TrackFS{
@@ -2026,7 +2116,7 @@ func TestArchiverAbortEarlyOnError(t *testing.T) {
 				SaveBlobConcurrency: 1,
 			})
 
-			_, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
+			_, _, _, err := arch.Snapshot(ctx, []string{"."}, SnapshotOptions{Time: time.Now()})
 			if !errors.Is(err, test.err) {
 				t.Errorf("expected error (%v) not found, got %v", test.err, err)
 			}
@@ -2054,7 +2144,7 @@ func snapshot(t testing.TB, repo restic.Repository, fs fs.FS, parent *restic.Sna
 		Time:           time.Now(),
 		ParentSnapshot: parent,
 	}
-	snapshot, _, err := arch.Snapshot(ctx, []string{filename}, sopts)
+	snapshot, _, _, err := arch.Snapshot(ctx, []string{filename}, sopts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -2125,6 +2215,8 @@ const (
 )
 
 func TestMetadataChanged(t *testing.T) {
+	defer feature.TestSetFlag(t, feature.Flag, feature.DeviceIDForHardlinks, true)()
+
 	files := TestDir{
 		"testfile": TestFile{
 			Content: "foo bar test file",
@@ -2133,12 +2225,12 @@ func TestMetadataChanged(t *testing.T) {
 
 	tempdir, repo := prepareTempdirRepoSrc(t, files)
 
-	back := restictest.Chdir(t, tempdir)
+	back := rtest.Chdir(t, tempdir)
 	defer back()
 
 	// get metadata
 	fi := lstat(t, "testfile")
-	want, err := restic.NodeFromFileInfo("testfile", fi)
+	want, err := restic.NodeFromFileInfo("testfile", fi, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -2153,6 +2245,7 @@ func TestMetadataChanged(t *testing.T) {
 	sn, node2 := snapshot(t, repo, fs, nil, "testfile")
 
 	// set some values so we can then compare the nodes
+	want.DeviceID = 0
 	want.Content = node2.Content
 	want.Path = ""
 	if len(want.ExtendedAttributes) == 0 {
@@ -2195,7 +2288,7 @@ func TestMetadataChanged(t *testing.T) {
 	// make sure the content matches
 	TestEnsureFileContent(context.Background(), t, repo, "testfile", node3, files["testfile"].(TestFile))
 
-	checker.TestCheckRepo(t, repo)
+	checker.TestCheckRepo(t, repo, false)
 }
 
 func TestRacyFileSwap(t *testing.T) {
@@ -2207,7 +2300,7 @@ func TestRacyFileSwap(t *testing.T) {
 
 	tempdir, repo := prepareTempdirRepoSrc(t, files)
 
-	back := restictest.Chdir(t, tempdir)
+	back := rtest.Chdir(t, tempdir)
 	defer back()
 
 	// get metadata of current folder
@@ -2236,7 +2329,7 @@ func TestRacyFileSwap(t *testing.T) {
 	arch.runWorkers(ctx, wg)
 
 	// fs.Track will panic if the file was not closed
-	_, excluded, err := arch.Save(ctx, "/", tempfile, nil)
+	_, excluded, err := arch.save(ctx, "/", tempfile, nil)
 	if err == nil {
 		t.Errorf("Save() should have failed")
 	}

internal/archiver/archiver_unix_test.go

@@ -6,6 +6,12 @@ package archiver
 import (
 	"os"
 	"syscall"
+	"testing"
+
+	"github.com/restic/restic/internal/feature"
+	"github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/restic"
+	rtest "github.com/restic/restic/internal/test"
 )
 
 type wrappedFileInfo struct {
@@ -39,3 +45,45 @@ func wrapFileInfo(fi os.FileInfo) os.FileInfo {
 
 	return res
 }
+
+func statAndSnapshot(t *testing.T, repo restic.Repository, name string) (*restic.Node, *restic.Node) {
+	fi := lstat(t, name)
+	want, err := restic.NodeFromFileInfo(name, fi, false)
+	rtest.OK(t, err)
+
+	_, node := snapshot(t, repo, fs.Local{}, nil, name)
+	return want, node
+}
+
+func TestHardlinkMetadata(t *testing.T) {
+	defer feature.TestSetFlag(t, feature.Flag, feature.DeviceIDForHardlinks, true)()
+
+	files := TestDir{
+		"testfile": TestFile{
+			Content: "foo bar test file",
+		},
+		"linktarget": TestFile{
+			Content: "test file",
+		},
+		"testlink": TestHardlink{
+			Target: "./linktarget",
+		},
+		"testdir": TestDir{},
+	}
+
+	tempdir, repo := prepareTempdirRepoSrc(t, files)
+
+	back := rtest.Chdir(t, tempdir)
+	defer back()
+
+	want, node := statAndSnapshot(t, repo, "testlink")
+	rtest.Assert(t, node.DeviceID == want.DeviceID, "device id mismatch expected %v got %v", want.DeviceID, node.DeviceID)
+	rtest.Assert(t, node.Links == want.Links, "link count mismatch expected %v got %v", want.Links, node.Links)
+	rtest.Assert(t, node.Inode == want.Inode, "inode mismatch expected %v got %v", want.Inode, node.Inode)
+
+	_, node = statAndSnapshot(t, repo, "testfile")
+	rtest.Assert(t, node.DeviceID == 0, "device id mismatch for testfile expected %v got %v", 0, node.DeviceID)
+
+	_, node = statAndSnapshot(t, repo, "testdir")
+	rtest.Assert(t, node.DeviceID == 0, "device id mismatch for testdir expected %v got %v", 0, node.DeviceID)
+}

internal/archiver/file_saver.go

@@ -29,7 +29,7 @@ type FileSaver struct {
 
 	CompleteBlob func(bytes uint64)
 
-	NodeFromFileInfo func(snPath, filename string, fi os.FileInfo) (*restic.Node, error)
+	NodeFromFileInfo func(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error)
 }
 
 // NewFileSaver returns a new file saver. A worker pool with fileWorkers is
@@ -156,7 +156,7 @@ func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat
 
 	debug.Log("%v", snPath)
 
-	node, err := s.NodeFromFileInfo(snPath, f.Name(), fi)
+	node, err := s.NodeFromFileInfo(snPath, f.Name(), fi, false)
 	if err != nil {
 		_ = f.Close()
 		completeError(err)

internal/archiver/file_saver_test.go

@@ -49,8 +49,8 @@ func startFileSaver(ctx context.Context, t testing.TB) (*FileSaver, context.Cont
 	}
 
 	s := NewFileSaver(ctx, wg, saveBlob, pol, workers, workers)
-	s.NodeFromFileInfo = func(snPath, filename string, fi os.FileInfo) (*restic.Node, error) {
-		return restic.NodeFromFileInfo(filename, fi)
+	s.NodeFromFileInfo = func(snPath, filename string, fi os.FileInfo, ignoreXattrListError bool) (*restic.Node, error) {
+		return restic.NodeFromFileInfo(filename, fi, ignoreXattrListError)
 	}
 
 	return s, ctx, wg

internal/archiver/scanner_test.go

@@ -9,7 +9,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/restic/restic/internal/fs"
-	restictest "github.com/restic/restic/internal/test"
+	rtest "github.com/restic/restic/internal/test"
 )
 
 func TestScanner(t *testing.T) {
@@ -81,10 +81,10 @@ func TestScanner(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.Background())
 			defer cancel()
 
-			tempdir := restictest.TempDir(t)
+			tempdir := rtest.TempDir(t)
 			TestCreateFiles(t, tempdir, test.src)
 
-			back := restictest.Chdir(t, tempdir)
+			back := rtest.Chdir(t, tempdir)
 			defer back()
 
 			cur, err := os.Getwd()
@@ -216,10 +216,10 @@ func TestScannerError(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.Background())
 			defer cancel()
 
-			tempdir := restictest.TempDir(t)
+			tempdir := rtest.TempDir(t)
 			TestCreateFiles(t, tempdir, test.src)
 
-			back := restictest.Chdir(t, tempdir)
+			back := rtest.Chdir(t, tempdir)
 			defer back()
 
 			cur, err := os.Getwd()
@@ -288,10 +288,10 @@ func TestScannerCancel(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	tempdir := restictest.TempDir(t)
+	tempdir := rtest.TempDir(t)
 	TestCreateFiles(t, tempdir, src)
 
-	back := restictest.Chdir(t, tempdir)
+	back := rtest.Chdir(t, tempdir)
 	defer back()
 
 	cur, err := os.Getwd()

internal/archiver/testing.go

@@ -6,6 +6,7 @@ import (
 	"path"
 	"path/filepath"
 	"runtime"
+	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -31,7 +32,7 @@ func TestSnapshot(t testing.TB, repo restic.Repository, path string, parent *res
 		}
 		opts.ParentSnapshot = sn
 	}
-	sn, _, err := arch.Snapshot(context.TODO(), []string{path}, opts)
+	sn, _, _, err := arch.Snapshot(context.TODO(), []string{path}, opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -63,11 +64,29 @@ func (s TestSymlink) String() string {
 	return "<Symlink>"
 }
 
+// TestHardlink describes a hardlink created for a test.
+type TestHardlink struct {
+	Target string
+}
+
+func (s TestHardlink) String() string {
+	return "<Hardlink>"
+}
+
 // TestCreateFiles creates a directory structure described by dir at target,
 // which must already exist. On Windows, symlinks aren't created.
 func TestCreateFiles(t testing.TB, target string, dir TestDir) {
 	t.Helper()
-	for name, item := range dir {
+
+	// ensure a stable order such that it can be guaranteed that a hardlink target already exists
+	var names []string
+	for name := range dir {
+		names = append(names, name)
+	}
+	sort.Strings(names)
+
+	for _, name := range names {
+		item := dir[name]
 		targetPath := filepath.Join(target, name)
 
 		switch it := item.(type) {
@@ -81,6 +100,11 @@ func TestCreateFiles(t testing.TB, target string, dir TestDir) {
 			if err != nil {
 				t.Fatal(err)
 			}
+		case TestHardlink:
+			err := fs.Link(filepath.Join(target, filepath.FromSlash(it.Target)), targetPath)
+			if err != nil {
+				t.Fatal(err)
+			}
 		case TestDir:
 			err := fs.Mkdir(targetPath, 0755)
 			if err != nil {

internal/archiver/testing_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/repository"
-	restictest "github.com/restic/restic/internal/test"
+	rtest "github.com/restic/restic/internal/test"
 )
 
 // MockT passes through all logging functions from T, but catches Fail(),
@@ -101,7 +101,7 @@ func TestTestCreateFiles(t *testing.T) {
 	}
 
 	for i, test := range tests {
-		tempdir := restictest.TempDir(t)
+		tempdir := rtest.TempDir(t)
 
 		t.Run("", func(t *testing.T) {
 			tempdir := filepath.Join(tempdir, fmt.Sprintf("test-%d", i))
@@ -191,7 +191,7 @@ func TestTestWalkFiles(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run("", func(t *testing.T) {
-			tempdir := restictest.TempDir(t)
+			tempdir := rtest.TempDir(t)
 
 			got := make(map[string]string)
 
@@ -321,7 +321,7 @@ func TestTestEnsureFiles(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run("", func(t *testing.T) {
-			tempdir := restictest.TempDir(t)
+			tempdir := rtest.TempDir(t)
 			createFilesAt(t, tempdir, test.files)
 
 			subtestT := testing.TB(t)
@@ -452,7 +452,7 @@ func TestTestEnsureSnapshot(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.Background())
 			defer cancel()
 
-			tempdir := restictest.TempDir(t)
+			tempdir := rtest.TempDir(t)
 
 			targetDir := filepath.Join(tempdir, "target")
 			err := fs.Mkdir(targetDir, 0700)
@@ -462,7 +462,7 @@ func TestTestEnsureSnapshot(t *testing.T) {
 
 			createFilesAt(t, targetDir, test.files)
 
-			back := restictest.Chdir(t, tempdir)
+			back := rtest.Chdir(t, tempdir)
 			defer back()
 
 			repo := repository.TestRepository(t)
@@ -473,7 +473,7 @@ func TestTestEnsureSnapshot(t *testing.T) {
 				Hostname: "localhost",
 				Tags:     []string{"test"},
 			}
-			_, id, err := arch.Snapshot(ctx, []string{"."}, opts)
+			_, id, _, err := arch.Snapshot(ctx, []string{"."}, opts)
 			if err != nil {
 				t.Fatal(err)
 			}

internal/archiver/tree_saver.go

@@ -90,6 +90,10 @@ func (s *TreeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, I
 		// return the error if it wasn't ignored
 		if fnr.err != nil {
 			debug.Log("err for %v: %v", fnr.snPath, fnr.err)
+			if fnr.err == context.Canceled {
+				return nil, stats, fnr.err
+			}
+
 			fnr.err = s.errFn(fnr.target, fnr.err)
 			if fnr.err == nil {
 				// ignore error

internal/archiver/tree_test.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/restic/restic/internal/fs"
-	restictest "github.com/restic/restic/internal/test"
+	rtest "github.com/restic/restic/internal/test"
 )
 
 // debug.Log requires Tree.String.
@@ -439,10 +439,10 @@ func TestTree(t *testing.T) {
 				t.Skip("skip test on unix")
 			}
 
-			tempdir := restictest.TempDir(t)
+			tempdir := rtest.TempDir(t)
 			TestCreateFiles(t, tempdir, test.src)
 
-			back := restictest.Chdir(t, tempdir)
+			back := rtest.Chdir(t, tempdir)
 			defer back()
 
 			tree, err := NewTree(fs.Local{}, test.targets)

internal/backend/http_transport.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/peterbourgon/unixtransport"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
 )
@@ -82,6 +83,8 @@ func Transport(opts TransportOptions) (http.RoundTripper, error) {
 		TLSClientConfig:       &tls.Config{},
 	}
 
+	unixtransport.Register(tr)
+
 	if opts.InsecureTLS {
 		tr.TLSClientConfig.InsecureSkipVerify = true
 	}

internal/backend/layout/layout.go

@@ -10,6 +10,7 @@ import (
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/feature"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
 )
@@ -93,6 +94,8 @@ func hasBackendFile(ctx context.Context, fs Filesystem, dir string) (bool, error
 // cannot be detected automatically.
 var ErrLayoutDetectionFailed = errors.New("auto-detecting the filesystem layout failed")
 
+var ErrLegacyLayoutFound = errors.New("detected legacy S3 layout. Use `RESTIC_FEATURES=deprecate-s3-legacy-layout=false restic migrate s3_layout` to migrate your repository")
+
 // DetectLayout tries to find out which layout is used in a local (or sftp)
 // filesystem at the given path. If repo is nil, an instance of LocalFilesystem
 // is used.
@@ -123,6 +126,10 @@ func DetectLayout(ctx context.Context, repo Filesystem, dir string) (Layout, err
 	}
 
 	if foundKeyFile && !foundKeysFile {
+		if feature.Flag.Enabled(feature.DeprecateS3LegacyLayout) {
+			return nil, ErrLegacyLayoutFound
+		}
+
 		debug.Log("found s3 layout at %v", dir)
 		return &S3LegacyLayout{
 			Path: dir,
@@ -145,6 +152,10 @@ func ParseLayout(ctx context.Context, repo Filesystem, layout, defaultLayout, pa
 			Join: repo.Join,
 		}
 	case "s3legacy":
+		if feature.Flag.Enabled(feature.DeprecateS3LegacyLayout) {
+			return nil, ErrLegacyLayoutFound
+		}
+
 		l = &S3LegacyLayout{
 			Path: path,
 			Join: repo.Join,

internal/backend/layout/layout_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/feature"
 	rtest "github.com/restic/restic/internal/test"
 )
 
@@ -352,6 +353,7 @@ func TestS3LegacyLayout(t *testing.T) {
 }
 
 func TestDetectLayout(t *testing.T) {
+	defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)()
 	path := rtest.TempDir(t)
 
 	var tests = []struct {
@@ -389,6 +391,7 @@ func TestDetectLayout(t *testing.T) {
 }
 
 func TestParseLayout(t *testing.T) {
+	defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)()
 	path := rtest.TempDir(t)
 
 	var tests = []struct {

internal/backend/local/config.go

@@ -10,7 +10,7 @@ import (
 // Config holds all information needed to open a local repository.
 type Config struct {
 	Path   string
-	Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"`
+	Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect) (deprecated)"`
 
 	Connections uint `option:"connections" help:"set a limit for the number of concurrent operations (default: 2)"`
 }

internal/backend/local/layout_test.go

@@ -6,10 +6,12 @@ import (
 	"testing"
 
 	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/feature"
 	rtest "github.com/restic/restic/internal/test"
 )
 
 func TestLayout(t *testing.T) {
+	defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)()
 	path := rtest.TempDir(t)
 
 	var tests = []struct {

internal/backend/mem/mem_backend.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/base64"
+	"fmt"
 	"hash"
 	"io"
 	"net/http"
@@ -41,7 +42,7 @@ func NewFactory() location.Factory {
 	)
 }
 
-var errNotFound = errors.New("not found")
+var errNotFound = fmt.Errorf("not found")
 
 const connectionCount = 2
 

internal/backend/rest/config_test.go

@@ -31,6 +31,13 @@ var configTests = []test.ConfigTestData[Config]{
 			Connections: 5,
 		},
 	},
+	{
+		S: "rest:http+unix:///tmp/rest.socket:/my_backup_repo/",
+		Cfg: Config{
+			URL:         parseURL("http+unix:///tmp/rest.socket:/my_backup_repo/"),
+			Connections: 5,
+		},
+	},
 }
 
 func TestParseConfig(t *testing.T) {

internal/backend/rest/rest_test.go

@@ -1,11 +1,18 @@
+//go:build go1.20
+// +build go1.20
+
 package rest_test
 
 import (
+	"bufio"
 	"context"
-	"net"
+	"fmt"
 	"net/url"
 	"os"
 	"os/exec"
+	"regexp"
+	"strings"
+	"syscall"
 	"testing"
 	"time"
 
@@ -14,54 +21,133 @@ import (
 	rtest "github.com/restic/restic/internal/test"
 )
 
-func runRESTServer(ctx context.Context, t testing.TB, dir string) (*url.URL, func()) {
+var (
+	serverStartedRE = regexp.MustCompile("^start server on (.*)$")
+)
+
+func runRESTServer(ctx context.Context, t testing.TB, dir, reqListenAddr string) (*url.URL, func()) {
 	srv, err := exec.LookPath("rest-server")
 	if err != nil {
 		t.Skip(err)
 	}
 
-	cmd := exec.CommandContext(ctx, srv, "--no-auth", "--path", dir)
+	// create our own context, so that our cleanup can cancel and wait for completion
+	// this will ensure any open ports, open unix sockets etc are properly closed
+	processCtx, cancel := context.WithCancel(ctx)
+	cmd := exec.CommandContext(processCtx, srv, "--no-auth", "--path", dir, "--listen", reqListenAddr)
+
+	// this cancel func is called by when the process context is done
+	cmd.Cancel = func() error {
+		// we execute in a Go-routine as we know the caller will
+		// be waiting on a .Wait() regardless
+		go func() {
+			// try to send a graceful termination signal
+			if cmd.Process.Signal(syscall.SIGTERM) == nil {
+				// if we succeed, then wait a few seconds
+				time.Sleep(2 * time.Second)
+			}
+			// and then make sure it's killed either way, ignoring any error code
+			_ = cmd.Process.Kill()
+		}()
+		return nil
+	}
+
+	// this is the cleanup function that we return the caller,
+	// which will cancel our process context, and then wait for it to finish
+	cleanup := func() {
+		cancel()
+		_ = cmd.Wait()
+	}
+
+	// but in-case we don't finish this method, e.g. by calling t.Fatal()
+	// we also defer a call to clean it up ourselves, guarded by a flag to
+	// indicate that we returned the function to the caller to deal with.
+	callerWillCleanUp := false
+	defer func() {
+		if !callerWillCleanUp {
+			cleanup()
+		}
+	}()
+
+	// send stdout to our std out
 	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stdout
-	if err := cmd.Start(); err != nil {
-		t.Fatal(err)
-	}
 
-	// wait until the TCP port is reachable
-	var success bool
-	for i := 0; i < 10; i++ {
-		time.Sleep(200 * time.Millisecond)
-
-		c, err := net.Dial("tcp", "localhost:8000")
-		if err != nil {
-			continue
-		}
-
-		success = true
-		if err := c.Close(); err != nil {
-			t.Fatal(err)
-		}
-	}
-
-	if !success {
-		t.Fatal("unable to connect to rest server")
-		return nil, nil
-	}
-
-	url, err := url.Parse("http://localhost:8000/restic-test/")
+	// capture stderr with a pipe, as we want to examine this output
+	// to determine when the server is started and listening.
+	cmdErr, err := cmd.StderrPipe()
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	cleanup := func() {
-		if err := cmd.Process.Kill(); err != nil {
-			t.Fatal(err)
-		}
-
-		// ignore errors, we've killed the process
-		_ = cmd.Wait()
+	// start the rest-server
+	if err := cmd.Start(); err != nil {
+		t.Fatal(err)
 	}
 
+	// create a channel to receive the actual listen address on
+	listenAddrCh := make(chan string)
+	go func() {
+		defer close(listenAddrCh)
+		matched := false
+		br := bufio.NewReader(cmdErr)
+		for {
+			line, err := br.ReadString('\n')
+			if err != nil {
+				// we ignore errors, as code that relies on this
+				// will happily fail via timeout and empty closed
+				// channel.
+				return
+			}
+
+			line = strings.Trim(line, "\r\n")
+			if !matched {
+				// look for the server started message, and return the address
+				// that it's listening on
+				matchedServerListen := serverStartedRE.FindSubmatch([]byte(line))
+				if len(matchedServerListen) == 2 {
+					listenAddrCh <- string(matchedServerListen[1])
+					matched = true
+				}
+			}
+			fmt.Fprintln(os.Stdout, line) // print all output to console
+		}
+	}()
+
+	// wait for us to get an address,
+	// or the parent context to cancel,
+	// or for us to timeout
+	var actualListenAddr string
+	select {
+	case <-processCtx.Done():
+		t.Fatal(context.Canceled)
+	case <-time.NewTimer(2 * time.Second).C:
+		t.Fatal(context.DeadlineExceeded)
+	case a, ok := <-listenAddrCh:
+		if !ok {
+			t.Fatal(context.Canceled)
+		}
+		actualListenAddr = a
+	}
+
+	// this translate the address that the server is listening on
+	// to a URL suitable for us to connect to
+	var addrToConnectTo string
+	if strings.HasPrefix(reqListenAddr, "unix:") {
+		addrToConnectTo = fmt.Sprintf("http+unix://%s:/restic-test/", actualListenAddr)
+	} else {
+		// while we may listen on 0.0.0.0, we connect to localhost
+		addrToConnectTo = fmt.Sprintf("http://%s/restic-test/", strings.Replace(actualListenAddr, "0.0.0.0", "localhost", 1))
+	}
+
+	// parse to a URL
+	url, err := url.Parse(addrToConnectTo)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// indicate that we've completed successfully, and that the caller
+	// is responsible for calling cleanup
+	callerWillCleanUp = true
 	return url, cleanup
 }
 
@@ -91,7 +177,7 @@ func TestBackendREST(t *testing.T) {
 	defer cancel()
 
 	dir := rtest.TempDir(t)
-	serverURL, cleanup := runRESTServer(ctx, t, dir)
+	serverURL, cleanup := runRESTServer(ctx, t, dir, ":0")
 	defer cleanup()
 
 	newTestSuite(serverURL, false).RunTests(t)
@@ -116,7 +202,7 @@ func BenchmarkBackendREST(t *testing.B) {
 	defer cancel()
 
 	dir := rtest.TempDir(t)
-	serverURL, cleanup := runRESTServer(ctx, t, dir)
+	serverURL, cleanup := runRESTServer(ctx, t, dir, ":0")
 	defer cleanup()
 
 	newTestSuite(serverURL, false).RunBenchmarks(t)

internal/backend/rest/rest_unix_test.go

@@ -0,0 +1,30 @@
+//go:build !windows && go1.20
+// +build !windows,go1.20
+
+package rest_test
+
+import (
+	"context"
+	"fmt"
+	"path"
+	"testing"
+
+	rtest "github.com/restic/restic/internal/test"
+)
+
+func TestBackendRESTWithUnixSocket(t *testing.T) {
+	defer func() {
+		if t.Skipped() {
+			rtest.SkipDisallowed(t, "restic/backend/rest.TestBackendREST")
+		}
+	}()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	dir := rtest.TempDir(t)
+	serverURL, cleanup := runRESTServer(ctx, t, path.Join(dir, "data"), fmt.Sprintf("unix:%s", path.Join(dir, "sock")))
+	defer cleanup()
+
+	newTestSuite(serverURL, false).RunTests(t)
+}

internal/backend/s3/config.go

@@ -20,7 +20,7 @@ type Config struct {
 	Secret       options.SecretString
 	Bucket       string
 	Prefix       string
-	Layout       string `option:"layout" help:"use this backend layout (default: auto-detect)"`
+	Layout       string `option:"layout" help:"use this backend layout (default: auto-detect) (deprecated)"`
 	StorageClass string `option:"storage-class" help:"set S3 storage class (STANDARD, STANDARD_IA, ONEZONE_IA, INTELLIGENT_TIERING or REDUCED_REDUNDANCY)"`
 
 	Connections   uint   `option:"connections" help:"set a limit for the number of concurrent connections (default: 5)"`

internal/backend/sftp/config.go

@@ -13,7 +13,7 @@ import (
 type Config struct {
 	User, Host, Port, Path string
 
-	Layout  string `option:"layout"  help:"use this backend directory layout (default: auto-detect)"`
+	Layout  string `option:"layout"  help:"use this backend directory layout (default: auto-detect) (deprecated)"`
 	Command string `option:"command" help:"specify command to create sftp connection"`
 	Args    string `option:"args"    help:"specify arguments for ssh"`
 

internal/backend/sftp/layout_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/backend/sftp"
+	"github.com/restic/restic/internal/feature"
 	rtest "github.com/restic/restic/internal/test"
 )
 
@@ -16,6 +17,7 @@ func TestLayout(t *testing.T) {
 		t.Skip("sftp server binary not available")
 	}
 
+	defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)()
 	path := rtest.TempDir(t)
 
 	var tests = []struct {

internal/cache/file.go

@@ -165,7 +165,8 @@ func (c *Cache) Clear(t restic.FileType, valid restic.IDSet) error {
 			continue
 		}
 
-		if err = fs.Remove(c.filename(backend.Handle{Type: t, Name: id.String()})); err != nil {
+		// ignore ErrNotExist to gracefully handle multiple processes running Clear() concurrently
+		if err = fs.Remove(c.filename(backend.Handle{Type: t, Name: id.String()})); err != nil && !errors.Is(err, os.ErrNotExist) {
 			return err
 		}
 	}

internal/checker/checker.go

@@ -106,9 +106,9 @@ func (c *Checker) LoadSnapshots(ctx context.Context) error {
 	return err
 }
 
-func computePackTypes(ctx context.Context, idx restic.MasterIndex) map[restic.ID]restic.BlobType {
+func computePackTypes(ctx context.Context, idx restic.MasterIndex) (map[restic.ID]restic.BlobType, error) {
 	packs := make(map[restic.ID]restic.BlobType)
-	idx.Each(ctx, func(pb restic.PackedBlob) {
+	err := idx.Each(ctx, func(pb restic.PackedBlob) {
 		tpe, exists := packs[pb.PackID]
 		if exists {
 			if pb.Type != tpe {
@@ -119,7 +119,7 @@ func computePackTypes(ctx context.Context, idx restic.MasterIndex) map[restic.ID
 		}
 		packs[pb.PackID] = tpe
 	})
-	return packs
+	return packs, err
 }
 
 // LoadIndex loads all index files.
@@ -169,7 +169,7 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e
 
 		debug.Log("process blobs")
 		cnt := 0
-		index.Each(ctx, func(blob restic.PackedBlob) {
+		err = index.Each(ctx, func(blob restic.PackedBlob) {
 			cnt++
 
 			if _, ok := packToIndex[blob.PackID]; !ok {
@@ -179,7 +179,7 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e
 		})
 
 		debug.Log("%d blobs processed", cnt)
-		return nil
+		return err
 	})
 	if err != nil {
 		errs = append(errs, err)
@@ -193,8 +193,14 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e
 	}
 
 	// compute pack size using index entries
-	c.packs = pack.Size(ctx, c.masterIndex, false)
-	packTypes := computePackTypes(ctx, c.masterIndex)
+	c.packs, err = pack.Size(ctx, c.masterIndex, false)
+	if err != nil {
+		return hints, append(errs, err)
+	}
+	packTypes, err := computePackTypes(ctx, c.masterIndex)
+	if err != nil {
+		return hints, append(errs, err)
+	}
 
 	debug.Log("checking for duplicate packs")
 	for packID := range c.packs {
@@ -484,7 +490,7 @@ func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) {
 }
 
 // UnusedBlobs returns all blobs that have never been referenced.
-func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles) {
+func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, err error) {
 	if !c.trackUnused {
 		panic("only works when tracking blob references")
 	}
@@ -495,7 +501,7 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles) {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	c.repo.Index().Each(ctx, func(blob restic.PackedBlob) {
+	err = c.repo.Index().Each(ctx, func(blob restic.PackedBlob) {
 		h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
 		if !c.blobRefs.M.Has(h) {
 			debug.Log("blob %v not referenced", h)
@@ -503,7 +509,7 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles) {
 		}
 	})
 
-	return blobs
+	return blobs, err
 }
 
 // CountPacks returns the number of packs in the repository.

internal/checker/checker_test.go

@@ -72,11 +72,9 @@ func assertOnlyMixedPackHints(t *testing.T, hints []error) {
 }
 
 func TestCheckRepo(t *testing.T) {
-	repodir, cleanup := test.Env(t, checkerTestData)
+	repo, cleanup := repository.TestFromFixture(t, checkerTestData)
 	defer cleanup()
 
-	repo := repository.TestOpenLocal(t, repodir)
-
 	chkr := checker.New(repo, false)
 	hints, errs := chkr.LoadIndex(context.TODO(), nil)
 	if len(errs) > 0 {
@@ -92,11 +90,9 @@ func TestCheckRepo(t *testing.T) {
 }
 
 func TestMissingPack(t *testing.T) {
-	repodir, cleanup := test.Env(t, checkerTestData)
+	repo, cleanup := repository.TestFromFixture(t, checkerTestData)
 	defer cleanup()
 
-	repo := repository.TestOpenLocal(t, repodir)
-
 	packHandle := backend.Handle{
 		Type: restic.PackFile,
 		Name: "657f7fb64f6a854fff6fe9279998ee09034901eded4e6db9bcee0e59745bbce6",
@@ -123,11 +119,9 @@ func TestMissingPack(t *testing.T) {
 }
 
 func TestUnreferencedPack(t *testing.T) {
-	repodir, cleanup := test.Env(t, checkerTestData)
+	repo, cleanup := repository.TestFromFixture(t, checkerTestData)
 	defer cleanup()
 
-	repo := repository.TestOpenLocal(t, repodir)
-
 	// index 3f1a only references pack 60e0
 	packID := "60e0438dcb978ec6860cc1f8c43da648170ee9129af8f650f876bad19f8f788e"
 	indexHandle := backend.Handle{
@@ -156,11 +150,9 @@ func TestUnreferencedPack(t *testing.T) {
 }
 
 func TestUnreferencedBlobs(t *testing.T) {
-	repodir, cleanup := test.Env(t, checkerTestData)
+	repo, cleanup := repository.TestFromFixture(t, checkerTestData)
 	defer cleanup()
 
-	repo := repository.TestOpenLocal(t, repodir)
-
 	snapshotHandle := backend.Handle{
 		Type: restic.SnapshotFile,
 		Name: "51d249d28815200d59e4be7b3f21a157b864dc343353df9d8e498220c2499b02",
@@ -188,18 +180,17 @@ func TestUnreferencedBlobs(t *testing.T) {
 	test.OKs(t, checkPacks(chkr))
 	test.OKs(t, checkStruct(chkr))
 
-	blobs := chkr.UnusedBlobs(context.TODO())
+	blobs, err := chkr.UnusedBlobs(context.TODO())
+	test.OK(t, err)
 	sort.Sort(blobs)
 
 	test.Equals(t, unusedBlobsBySnapshot, blobs)
 }
 
 func TestModifiedIndex(t *testing.T) {
-	repodir, cleanup := test.Env(t, checkerTestData)
+	repo, cleanup := repository.TestFromFixture(t, checkerTestData)
 	defer cleanup()
 
-	repo := repository.TestOpenLocal(t, repodir)
-
 	done := make(chan struct{})
 	defer close(done)
 
@@ -274,11 +265,9 @@ func TestModifiedIndex(t *testing.T) {
 var checkerDuplicateIndexTestData = filepath.Join("testdata", "duplicate-packs-in-index-test-repo.tar.gz")
 
 func TestDuplicatePacksInIndex(t *testing.T) {
-	repodir, cleanup := test.Env(t, checkerDuplicateIndexTestData)
+	repo, cleanup := repository.TestFromFixture(t, checkerDuplicateIndexTestData)
 	defer cleanup()
 
-	repo := repository.TestOpenLocal(t, repodir)
-
 	chkr := checker.New(repo, false)
 	hints, errs := chkr.LoadIndex(context.TODO(), nil)
 	if len(hints) == 0 {
@@ -342,9 +331,7 @@ func TestCheckerModifiedData(t *testing.T) {
 	t.Logf("archived as %v", sn.ID().Str())
 
 	beError := &errorBackend{Backend: repo.Backend()}
-	checkRepo, err := repository.New(beError, repository.Options{})
-	test.OK(t, err)
-	test.OK(t, checkRepo.SearchKey(context.TODO(), test.TestPassword, 5, ""))
+	checkRepo := repository.TestOpenBackend(t, beError)
 
 	chkr := checker.New(checkRepo, false)
 
@@ -399,10 +386,8 @@ func (r *loadTreesOnceRepository) LoadTree(ctx context.Context, id restic.ID) (*
 }
 
 func TestCheckerNoDuplicateTreeDecodes(t *testing.T) {
-	repodir, cleanup := test.Env(t, checkerTestData)
+	repo, cleanup := repository.TestFromFixture(t, checkerTestData)
 	defer cleanup()
-
-	repo := repository.TestOpenLocal(t, repodir)
 	checkRepo := &loadTreesOnceRepository{
 		Repository:  repo,
 		loadedTrees: restic.NewIDSet(),
@@ -549,9 +534,7 @@ func TestCheckerBlobTypeConfusion(t *testing.T) {
 }
 
 func loadBenchRepository(t *testing.B) (*checker.Checker, restic.Repository, func()) {
-	repodir, cleanup := test.Env(t, checkerTestData)
-
-	repo := repository.TestOpenLocal(t, repodir)
+	repo, cleanup := repository.TestFromFixture(t, checkerTestData)
 
 	chkr := checker.New(repo, false)
 	hints, errs := chkr.LoadIndex(context.TODO(), nil)

internal/checker/testing.go

@@ -8,7 +8,7 @@ import (
 )
 
 // TestCheckRepo runs the checker on repo.
-func TestCheckRepo(t testing.TB, repo restic.Repository) {
+func TestCheckRepo(t testing.TB, repo restic.Repository, skipStructure bool) {
 	chkr := New(repo, true)
 
 	hints, errs := chkr.LoadIndex(context.TODO(), nil)
@@ -33,18 +33,23 @@ func TestCheckRepo(t testing.TB, repo restic.Repository) {
 		t.Error(err)
 	}
 
-	// structure
-	errChan = make(chan error)
-	go chkr.Structure(context.TODO(), nil, errChan)
+	if !skipStructure {
+		// structure
+		errChan = make(chan error)
+		go chkr.Structure(context.TODO(), nil, errChan)
 
-	for err := range errChan {
-		t.Error(err)
-	}
+		for err := range errChan {
+			t.Error(err)
+		}
 
-	// unused blobs
-	blobs := chkr.UnusedBlobs(context.TODO())
-	if len(blobs) > 0 {
-		t.Errorf("unused blobs found: %v", blobs)
+		// unused blobs
+		blobs, err := chkr.UnusedBlobs(context.TODO())
+		if err != nil {
+			t.Error(err)
+		}
+		if len(blobs) > 0 {
+			t.Errorf("unused blobs found: %v", blobs)
+		}
 	}
 
 	// read data

internal/dump/common_test.go

@@ -78,7 +78,7 @@ func WriteTest(t *testing.T, format string, cd CheckDump) {
 			back := rtest.Chdir(t, tmpdir)
 			defer back()
 
-			sn, _, err := arch.Snapshot(ctx, []string{"."}, archiver.SnapshotOptions{})
+			sn, _, _, err := arch.Snapshot(ctx, []string{"."}, archiver.SnapshotOptions{})
 			rtest.OK(t, err)
 
 			tree, err := restic.LoadTree(ctx, repo, *sn.Tree)

internal/feature/registry.go

@@ -5,11 +5,15 @@ var Flag = New()
 
 // flag names are written in kebab-case
 const (
-	ExampleFeature FlagName = "example-feature"
+	DeprecateLegacyIndex    FlagName = "deprecate-legacy-index"
+	DeprecateS3LegacyLayout FlagName = "deprecate-s3-legacy-layout"
+	DeviceIDForHardlinks    FlagName = "device-id-for-hardlinks"
 )
 
 func init() {
 	Flag.SetFlags(map[FlagName]FlagDesc{
-		ExampleFeature: {Type: Alpha, Description: "just for testing"},
+		DeprecateLegacyIndex:    {Type: Beta, Description: "disable support for index format used by restic 0.1.0. Use `restic repair index` to update the index if necessary."},
+		DeprecateS3LegacyLayout: {Type: Beta, Description: "disable support for S3 legacy layout used up to restic 0.7.0. Use `RESTIC_FEATURES=deprecate-s3-legacy-layout=false restic migrate s3_layout` to migrate your S3 repository if necessary."},
+		DeviceIDForHardlinks:    {Type: Alpha, Description: "store deviceID only for hardlinks to reduce metadata changes for example when using btrfs subvolumes. Will be removed in a future restic version after repository format 3 is available"},
 	})
 }