Merge pull request #4809 from MichaelEischer/update-changelog

add retries for corrupted blobs to changelog

.github/workflows/docker.yml

@@ -25,7 +25,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Log in to the Container registry
-        uses: docker/login-action@3d58c274f17dffee475a5520cbe67f0a882c4dbb
+        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -33,7 +33,7 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
+        uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
@@ -45,7 +45,7 @@ jobs:
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226
+        uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20
 
       - name: Ensure consistent binaries
         run: |
@@ -55,7 +55,7 @@ jobs:
         if: github.ref != 'refs/heads/master'
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
+        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0
         with:
           push: true
           context: .

.github/workflows/tests.yml

@@ -13,7 +13,7 @@ permissions:
   contents: read
 
 env:
-  latest_go: "1.21.x"
+  latest_go: "1.22.x"
   GO111MODULE: on
 
 jobs:
@@ -23,27 +23,32 @@ jobs:
         # list of jobs to run:
         include:
           - job_name: Windows
-            go: 1.21.x
+            go: 1.22.x
             os: windows-latest
 
           - job_name: macOS
-            go: 1.21.x
+            go: 1.22.x
             os: macOS-latest
             test_fuse: false
 
           - job_name: Linux
-            go: 1.21.x
+            go: 1.22.x
             os: ubuntu-latest
             test_cloud_backends: true
             test_fuse: true
             check_changelog: true
 
           - job_name: Linux (race)
-            go: 1.21.x
+            go: 1.22.x
             os: ubuntu-latest
             test_fuse: true
             test_opts: "-race"
 
+          - job_name: Linux
+            go: 1.21.x
+            os: ubuntu-latest
+            test_fuse: true
+
           - job_name: Linux
             go: 1.20.x
             os: ubuntu-latest
@@ -69,7 +74,7 @@ jobs:
       - name: Get programs (Linux/macOS)
         run: |
           echo "build Go tools"
-          go install github.com/restic/rest-server/cmd/rest-server@latest
+          go install github.com/restic/rest-server/cmd/rest-server@master
 
           echo "install minio server"
           mkdir $HOME/bin
@@ -101,7 +106,7 @@ jobs:
           $ProgressPreference = 'SilentlyContinue'
 
           echo "build Go tools"
-          go install github.com/restic/rest-server/...
+          go install github.com/restic/rest-server/cmd/rest-server@master
 
           echo "install minio server"
           mkdir $Env:USERPROFILE/bin
@@ -242,6 +247,10 @@ jobs:
   lint:
     name: lint
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      # allow annotating code in the PR
+      checks: write
     steps:
       - name: Set up Go ${{ env.latest_go }}
         uses: actions/setup-go@v5
@@ -252,10 +261,10 @@ jobs:
         uses: actions/checkout@v4
 
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v5
         with:
           # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
-          version: v1.55.2
+          version: v1.57.1
           args: --verbose --timeout 5m
 
         # only run golangci-lint for pull requests, otherwise ALL hints get
@@ -293,7 +302,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           # list of Docker images to use as base name for tags
           images: |
@@ -316,7 +325,7 @@ jobs:
 
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           push: false
           context: .

.gitignore

@@ -1,3 +1,4 @@
+/.idea
 /restic
 /restic.exe
 /.vagrant

.golangci.yml

@@ -35,6 +35,11 @@ linters:
     # parse and typecheck code
     - typecheck
 
+    # ensure that http response bodies are closed
+    - bodyclose
+
+    - importas
+
 issues:
   # don't use the default exclude rules, this hides (among others) ignored
   # errors from Close() calls
@@ -51,3 +56,14 @@ issues:
     # staticcheck: there's no easy way to replace these packages
     - "SA1019: \"golang.org/x/crypto/poly1305\" is deprecated"
     - "SA1019: \"golang.org/x/crypto/openpgp\" is deprecated"
+
+  exclude-rules:
+    # revive: ignore unused parameters in tests
+    - path: (_test\.go|testing\.go|backend/.*/tests\.go)
+      text: "unused-parameter:"
+
+linters-settings:
+  importas:
+    alias:
+      - pkg: github.com/restic/restic/internal/test
+        alias: rtest

.readthedocs.yaml

@@ -8,6 +8,10 @@ build:
   tools:
     python: "3.11"
 
+# Build HTMLZip
+formats:
+  - htmlzip
+
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
   configuration: doc/conf.py

CHANGELOG.md

@@ -3488,7 +3488,7 @@ restic users. The changes are ordered by importance.
 
    NOTE: This new implementation does not guarantee order in which blobs are
    written to the target files and, for example, the last blob of a file can be
-   written to the file before any of the preceeding file blobs. It is therefore
+   written to the file before any of the preceding file blobs. It is therefore
    possible to have gaps in the data written to the target files if restore fails
    or interrupted by the user.
 

CONTRIBUTING.md

@@ -6,7 +6,8 @@ Ways to Help Out
 Thank you for your contribution! Please **open an issue first** (or add a
 comment to an existing issue) if you plan to work on any code or add a new
 feature. This way, duplicate work is prevented and we can discuss your ideas
-and design first.
+and design first. Small bugfixes are an exception to this rule, just open a
+pull request in this case.
 
 There are several ways you can help us out. First of all code contributions and
 bug fixes are most welcome. However even "minor" details as fixing spelling
@@ -61,7 +62,7 @@ uploading it somewhere or post only the parts that are really relevant.
 If restic gets stuck, please also include a stacktrace in the description.
 On non-Windows systems, you can send a SIGQUIT signal to restic or press
 `Ctrl-\` to achieve the same result. This causes restic to print a stacktrace
-and then exit immediatelly. This will not damage your repository, however,
+and then exit immediately. This will not damage your repository, however,
 it might be necessary to manually clean up stale lock files using
 `restic unlock`.
 

README.md

@@ -10,8 +10,7 @@ For detailed usage and installation instructions check out the [documentation](h
 
 You can ask questions in our [Discourse forum](https://forum.restic.net).
 
-Quick start
------------
+## Quick start
 
 Once you've [installed](https://restic.readthedocs.io/en/latest/020_installation.html) restic, start
 off with creating a repository for your backups:
@@ -59,7 +58,7 @@ Therefore, restic supports the following backends for storing backups natively:
 Restic is a program that does backups right and was designed with the
 following principles in mind:
 
--  **Easy:** Doing backups should be a frictionless process, otherwise
+-  **Easy**: Doing backups should be a frictionless process, otherwise
    you might be tempted to skip it. Restic should be easy to configure
    and use, so that, in the event of a data loss, you can just restore
    it. Likewise, restoring data should not be complicated.
@@ -92,20 +91,17 @@ reproduce a byte identical version from the source code for that
 release. Instructions on how to do that are contained in the
 [builder repository](https://github.com/restic/builder).
 
-News
-----
+## News
 
-You can follow the restic project on Mastodon [@resticbackup](https://fosstodon.org/@restic) or by subscribing to
+You can follow the restic project on Mastodon [@resticbackup](https://fosstodon.org/@restic) or subscribe to
 the [project blog](https://restic.net/blog/).
 
-License
--------
+## License
 
 Restic is licensed under [BSD 2-Clause License](https://opensource.org/licenses/BSD-2-Clause). You can find the
-complete text in [``LICENSE``](LICENSE).
+complete text in [`LICENSE`](LICENSE).
 
-Sponsorship
------------
+## Sponsorship
 
 Backend integration tests for Google Cloud Storage and Microsoft Azure Blob
 Storage are sponsored by [AppsCode](https://appscode.com)!

changelog/0.10.0_2020-09-19/pull-2195

@@ -10,7 +10,7 @@ https://github.com/restic/restic/issues/2244
 
 NOTE: This new implementation does not guarantee order in which blobs
 are written to the target files and, for example, the last blob of a
-file can be written to the file before any of the preceeding file blobs.
+file can be written to the file before any of the preceding file blobs.
 It is therefore possible to have gaps in the data written to the target
 files if restore fails or interrupted by the user.
 

changelog/0.10.0_2020-09-19/pull-2668

@@ -1,6 +1,6 @@
 Bugfix: Don't abort the stats command when data blobs are missing
 
-Runing the stats command in the blobs-per-file mode on a repository with
+Running the stats command in the blobs-per-file mode on a repository with
 missing data blobs previously resulted in a crash.
 
 https://github.com/restic/restic/pull/2668

changelog/0.11.0_2020-11-05/issue-1756

@@ -1,6 +1,6 @@
 Bugfix: Mark repository files as read-only when using the local backend
 
-Files stored in a local repository were marked as writeable on the
+Files stored in a local repository were marked as writable on the
 filesystem for non-Windows systems, which did not prevent accidental file
 modifications outside of restic. In addition, the local backend did not work
 with certain filesystems and network mounts which do not permit modifications

changelog/0.11.0_2020-11-05/issue-340

@@ -5,7 +5,7 @@ another process using an exclusive lock through a filesystem snapshot. Restic
 was unable to backup those files before. This update enables backing up these
 files.
 
-This needs to be enabled explicitely using the --use-fs-snapshot option of the
+This needs to be enabled explicitly using the --use-fs-snapshot option of the
 backup command.
 
 https://github.com/restic/restic/issues/340

changelog/0.12.0_2021-02-14/pull-3106

@@ -2,7 +2,7 @@ Enhancement: Parallelize scan of snapshot content in `copy` and `prune`
 
 The `copy` and `prune` commands used to traverse the directories of
 snapshots one by one to find used data. This snapshot traversal is
-now parallized which can speed up this step several times.
+now parallelized which can speed up this step several times.
 
 In addition the `check` command now reports how many snapshots have
 already been processed.

changelog/0.8.0_2017-11-26/pull-1040

@@ -3,7 +3,7 @@ Enhancement: Add local metadata cache
 We've added a local cache for metadata so that restic doesn't need to load
 all metadata (snapshots, indexes, ...) from the repo each time it starts. By
 default the cache is active, but there's a new global option `--no-cache`
-that can be used to disable the cache. By deafult, the cache a standard
+that can be used to disable the cache. By default, the cache a standard
 cache folder for the OS, which can be overridden with `--cache-dir`.  The
 cache will automatically populate, indexes and snapshots are saved as they
 are loaded. Cache directories for repos that haven't been used recently can

changelog/0.8.0_2017-11-26/pull-1319

@@ -1,6 +1,6 @@
 Enhancement: Make `check` print `no errors found` explicitly
 
-The `check` command now explicetly prints `No errors were found` when no errors
+The `check` command now explicitly prints `No errors were found` when no errors
 could be found.
 
 https://github.com/restic/restic/pull/1319

changelog/0.8.2_2018-02-17/issue-1506

@@ -1,4 +1,4 @@
-Bugfix: Limit bandwith at the http.RoundTripper for HTTP based backends
+Bugfix: Limit bandwidth at the http.RoundTripper for HTTP based backends
 
 https://github.com/restic/restic/issues/1506
 https://github.com/restic/restic/pull/1511

changelog/0.8.2_2018-02-17/pull-1595

@@ -1,7 +1,7 @@
 Bugfix: backup: Remove bandwidth display
 
 This commit removes the bandwidth displayed during backup process. It is
-misleading and seldomly correct, because it's neither the "read
+misleading and seldom correct, because it's neither the "read
 bandwidth" (only for the very first backup) nor the "upload bandwidth".
 Many users are confused about (and rightly so), c.f. #1581, #1033, #1591
 

changelog/0.8.3_2018-02-26/pull-1623

@@ -6,7 +6,7 @@ that means making a request (e.g. via HTTP) and returning an error when the
 file already exists.
 
 This is not accurate, the file could have been created between the HTTP request
-testing for it, and when writing starts, so we've relaxed this requeriment,
+testing for it, and when writing starts, so we've relaxed this requirement,
 which saves one additional HTTP request per newly added file.
 
 https://github.com/restic/restic/pull/1623

changelog/0.9.0_2018-05-21/pull-1735

@@ -1,4 +1,4 @@
-Enhancement: Allow keeping a time range of snaphots
+Enhancement: Allow keeping a time range of snapshots
 
 We've added the `--keep-within` option to the `forget` command. It instructs
 restic to keep all snapshots within the given duration since the newest

changelog/0.9.3_2018-10-13/pull-1876

@@ -1,7 +1,7 @@
 Enhancement: Display reason why forget keeps snapshots
 
 We've added a column to the list of snapshots `forget` keeps which details the
-reasons to keep a particuliar snapshot. This makes debugging policies for
+reasons to keep a particular snapshot. This makes debugging policies for
 forget much easier. Please remember to always try things out with `--dry-run`!
 
 https://github.com/restic/restic/pull/1876

changelog/0.9.6_2019-11-22/issue-2179

@@ -9,7 +9,7 @@ file should be noticed, and the modified file will be backed up. The ctime check
 will be disabled if the --ignore-inode flag was given.
 
 If this change causes problems for you, please open an issue, and we can look in
-to adding a seperate flag to disable just the ctime check.
+to adding a separate flag to disable just the ctime check.
 
 https://github.com/restic/restic/issues/2179
 https://github.com/restic/restic/pull/2212

changelog/unreleased/issue-3600

@@ -0,0 +1,11 @@
+Bugfix: `backup` works if xattrs above the backup target cannot be read
+
+When backup targets are specified using absolute paths, then `backup` also
+includes information about the parent folders of the backup targets in the
+snapshot. If the extended attributes for some of these folders could not be
+read due to missing permissions, this caused the backup to fail. This has been
+fixed.
+
+https://github.com/restic/restic/issues/3600
+https://github.com/restic/restic/pull/4668
+https://forum.restic.net/t/parent-directories-above-the-snapshot-source-path-fatal-error-permission-denied/7216

changelog/unreleased/issue-4209

@@ -0,0 +1,7 @@
+Bugfix: Fix slow sftp upload performance
+
+Since restic 0.12.1, the upload speed of the sftp backend to a remote server
+has regressed significantly. This has been fixed.
+
+https://github.com/restic/restic/issues/4209
+https://github.com/restic/restic/pull/4782

changelog/unreleased/issue-4251

@@ -0,0 +1,16 @@
+Enhancement: Support reading backup from a commands's standard output
+
+The `backup` command now supports the `--stdin-from-command` option. When using
+this option, the arguments to `backup` are interpreted as a command instead of
+paths to back up. `backup` then executes the given command and stores the
+standard output from it in the backup, similar to the what the `--stdin` option
+does. This also enables restic to verify that the command completes with exit
+code zero. A non-zero exit code causes the backup to fail.
+
+Note that the `--stdin` option does not have to be specified at the same time,
+and that the `--stdin-filename` option also applies to `--stdin-from-command`.
+
+Example: `restic backup --stdin-from-command --stdin-filename dump.sql mysqldump [...]`
+
+https://github.com/restic/restic/issues/4251
+https://github.com/restic/restic/pull/4410

changelog/unreleased/issue-4287

@@ -0,0 +1,14 @@
+Enhancement: support connection to rest-server using unix socket
+
+Restic now supports connecting to rest-server using a unix socket for
+rest-server version 0.13.0 or later.
+
+This allows running restic as follows:
+
+```
+rest-server --listen unix:/tmp/rest.socket --data /path/to/data &
+restic -r rest:http+unix:///tmp/rest.socket:/my_backup_repo/ [...]
+```
+
+https://github.com/restic/restic/issues/4287
+https://github.com/restic/restic/pull/4655

changelog/unreleased/issue-4472

@@ -0,0 +1,18 @@
+Enhancement: Allow AWS Assume Role to be used for S3 backend
+
+Previously only credentials discovered via the Minio discovery methods
+were used to authenticate.
+
+However, there are many circumstances where the discovered credentials have
+lower permissions and need to assume a specific role. This is now possible
+using the following new environment variables.
+
+- RESTIC_AWS_ASSUME_ROLE_ARN
+- RESTIC_AWS_ASSUME_ROLE_SESSION_NAME
+- RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID
+- RESTIC_AWS_ASSUME_ROLE_REGION (defaults to us-east-1)
+- RESTIC_AWS_ASSUME_ROLE_POLICY
+- RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT
+
+https://github.com/restic/restic/issues/4472
+https://github.com/restic/restic/pull/4474

changelog/unreleased/issue-4540

@@ -0,0 +1,6 @@
+Change: Require at least ARMv6 for ARM binaries
+
+The official release binaries of restic now require at least ARMv6 support for ARM platforms.
+
+https://github.com/restic/restic/issues/4540
+https://github.com/restic/restic/pull/4542

changelog/unreleased/issue-4547

@@ -0,0 +1,7 @@
+Enhancement: Add support for `--json` option to `version` command
+
+Restic now supports outputting restic version and used go version, OS and
+architecture via JSON when using the version command.
+
+https://github.com/restic/restic/issues/4547
+https://github.com/restic/restic/pull/4553

changelog/unreleased/issue-4549

@@ -0,0 +1,11 @@
+Enhancement: Add `--ncdu` option to `ls` command
+
+NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories.
+It has an option to save a directory tree and analyse it later.
+The `ls` command now supports the `--ncdu` option which outputs information
+about a snapshot in the NCDU format.
+
+You can use it as follows: `restic ls latest --ncdu | ncdu -f -`
+
+https://github.com/restic/restic/issues/4549
+https://github.com/restic/restic/pull/4550

changelog/unreleased/issue-4583

@@ -0,0 +1,12 @@
+Enhancement: Ignore s3.storage-class for metadata if archive tier is specified
+
+There is no official cold storage support in restic, use this option at your
+own risk.
+
+Restic always stored all files on s3 using the specified `s3.storage-class`.
+Now, restic will store metadata using a non-archive storage tier to avoid
+problems when accessing a repository. To restore any data, it is still
+necessary to manually warm up the required data beforehand.
+
+https://github.com/restic/restic/issues/4583
+https://github.com/restic/restic/pull/4584

changelog/unreleased/issue-4601

@@ -0,0 +1,9 @@
+Enhancement: Add support for feature flags
+
+Restic now supports feature flags that can be used to enable and disable
+experimental features. The flags can be set using the environment variable
+`RESTIC_FEATURES`. To get a list of currently supported feature flags,
+run the `features` command.
+
+https://github.com/restic/restic/issues/4601
+https://github.com/restic/restic/pull/4666

changelog/unreleased/issue-4602

@@ -0,0 +1,23 @@
+Change: Deprecate legacy index format and s3legacy layout
+
+Support for the legacy index format used by restic before version 0.2.0 has
+been deprecated and will be removed in the next minor restic version. You can
+use `restic repair index` to update the index to the current format.
+
+It is possible to temporarily reenable support for the legacy index format by
+setting the environment variable
+`RESTIC_FEATURES=deprecate-legacy-index=false`. Note that this feature flag
+will be removed in the next minor restic version.
+
+Support for the s3legacy layout used for the S3 backend before restic 0.7.0
+has been deprecated and will be removed in the next minor restic version. You
+can migrate your S3 repository using `RESTIC_FEATURES=deprecate-s3-legacy-layout=false restic migrate s3_layout`.
+
+It is possible to temporarily reenable support for the legacy s3layout by
+setting the environment variable
+`RESTIC_FEATURES=deprecate-s3-legacy-layout=false`. Note that this feature flag
+will be removed in the next minor restic version.
+
+https://github.com/restic/restic/issues/4602
+https://github.com/restic/restic/pull/4724
+https://github.com/restic/restic/pull/4743

changelog/unreleased/issue-4627

@@ -0,0 +1,28 @@
+Change: Redesign backend error handling to improve reliability
+
+Restic now downloads pack files in large chunks instead of using a streaming
+download. This prevents failures due to interrupted streams. The `restore`
+command now also retries downloading individual blobs that cannot be retrieved.
+
+HTTP requests that are stuck for more than two minutes while uploading or
+downloading are now forcibly interrupted. This ensures that stuck requests are
+retried after a short timeout.
+
+Attempts to access a missing file or a truncated file will no longer be retried.
+This avoids unnecessary retries in those cases.
+
+If a download yields a corrupt file or blob, then the download will be retried once.
+
+Most parts of the new backend error handling can temporarily be disabled by
+setting the environment variable
+`RESTIC_FEATURES=backend-error-redesign=false`. Note that this feature flag will
+be removed in the next minor restic version.
+
+https://github.com/restic/restic/issues/4627
+https://github.com/restic/restic/issues/4193
+https://github.com/restic/restic/pull/4605
+https://github.com/restic/restic/pull/4792
+https://github.com/restic/restic/issues/4515
+https://github.com/restic/restic/issues/1523
+https://github.com/restic/restic/pull/4520
+https://github.com/restic/restic/pull/4800

changelog/unreleased/issue-4656

@@ -0,0 +1,7 @@
+Bugfix: Properly report the ID of newly added keys
+
+`restic key add` now reports the ID of a newly added key. This simplifies
+selecting a specific key using the `--key-hint key` option.
+
+https://github.com/restic/restic/issues/4656
+https://github.com/restic/restic/pull/4657

changelog/unreleased/issue-4676

@@ -0,0 +1,8 @@
+Enhancement: Move key add, list, remove and passwd as separate sub-commands
+
+Restic now provides usage documentation for the `key` command. Each sub-command;
+`add`, `list`, `remove` and `passwd` now have their own sub-command documentation
+which can be invoked using `restic key <add|list|remove|passwd> --help`.
+
+https://github.com/restic/restic/issues/4676
+https://github.com/restic/restic/pull/4685

changelog/unreleased/issue-4678

@@ -0,0 +1,8 @@
+Enhancement: Add --target flag to the dump command
+
+Restic `dump` always printed to the standard output. It now permits to select a
+`--target` file to write the output to.
+
+https://github.com/restic/restic/issues/4678
+https://github.com/restic/restic/pull/4682
+https://github.com/restic/restic/pull/4692

changelog/unreleased/issue-4733

@@ -0,0 +1,10 @@
+Enhancement: Allow specifying `--host` via environment variable
+
+Restic commands that operate on snapshots, such as `restic backup` and
+`restic snapshots`, support the `--host` flag to specify the hostname for
+grouoping snapshots. They now permit selecting the hostname via the
+environment variable `RESTIC_HOST`. `--host` still takes precedence over the
+environment variable.
+
+https://github.com/restic/restic/issues/4733
+https://github.com/restic/restic/pull/4734

changelog/unreleased/issue-4744

@@ -0,0 +1,9 @@
+Change: Include full key ID in JSON output of `key list`
+
+We have changed the JSON output of the `key list` command to include the full
+key ID instead of just a shortened version, as the latter can be ambiguous
+in some rare cases. To derive the short ID, please truncate the full ID down to
+eight characters.
+
+https://github.com/restic/restic/issues/4744
+https://github.com/restic/restic/pull/4745

changelog/unreleased/issue-4760

@@ -0,0 +1,8 @@
+Bugfix: Fix possible error on concurrent cache cleanup
+
+If multiple restic processes concurrently cleaned up no longer existing files
+from the cache, this could cause some of the processes to fail with an `no such
+file or directory` error. This has been fixed.
+
+https://github.com/restic/restic/issues/4760
+https://github.com/restic/restic/pull/4761

changelog/unreleased/issue-693

@@ -0,0 +1,12 @@
+Enhancement: Support printing snapshot size in `snapshots` command
+
+The `snapshots` command now supports printing the snapshot size for snapshots
+created using this or a future restic version. For this, the `backup` command
+now stores the backup summary statistics in the snapshot.
+
+The text output of the `snapshots` command only shows the snapshot size. The
+other statistics are only included in the JSON output. To inspect these
+statistics use `restic snapshots --json` or `restic cat snapshot <snapshotID>`.
+
+https://github.com/restic/restic/issues/693
+https://github.com/restic/restic/pull/4705

changelog/unreleased/pull-3067

@@ -0,0 +1,22 @@
+Enhancement: Add options to configure Windows Shadow Copy Service
+
+Restic always used 120 seconds timeout and unconditionally created VSS snapshots
+for all volume mount points on disk. Now this behavior can be fine-tuned by
+new options, like exclude specific volumes and mount points or completely
+disable auto snapshotting of volume mount points.
+
+For example:
+
+    restic backup --use-fs-snapshot -o vss.timeout=5m -o vss.exclude-all-mount-points=true
+
+changes timeout to five minutes and disable snapshotting of mount points on all volumes, and
+
+    restic backup --use-fs-snapshot -o vss.exclude-volumes="d:\;c:\mnt\;\\?\Volume{e2e0315d-9066-4f97-8343-eb5659b35762}"
+
+excludes drive `d:`, mount point `c:\mnt` and specific volume from VSS snapshotting.
+
+    restic backup --use-fs-snapshot -o vss.provider={b5946137-7b9f-4925-af80-51abd60b20d5}
+
+uses 'Microsoft Software Shadow Copy provider 1.0' instead of the default provider.
+
+https://github.com/restic/restic/pull/3067

changelog/unreleased/pull-4006

@@ -0,0 +1,16 @@
+Enhancement: (alpha) Store deviceID only for hardlinks
+
+Set `RESTIC_FEATURES=device-id-for-hardlinks` to enable this alpha feature.
+The feature flag will be removed after repository format version 3 becomes
+available or be replaced with a different solution.
+
+When creating backups from a filesystem snapshot, for example created using
+btrfs subvolumes, the deviceID of the filesystem changes compared to previous
+snapshots. This prevented restic from deduplicating the directory metadata of
+a snapshot.
+
+When this alpha feature is enabled, then the deviceID is only stored for
+hardlinks. This significantly reduces the metadata duplication for most
+backups.
+
+https://github.com/restic/restic/pull/4006

changelog/unreleased/pull-4503

@@ -0,0 +1,8 @@
+Bugfix: Correct hardlink handling in `stats` command
+
+If files on different devices had the same inode id, then the `stats` command
+did not correctly calculate the snapshot size. This has been fixed.
+
+https://github.com/restic/restic/pull/4503
+https://github.com/restic/restic/pull/4006
+https://forum.restic.net/t/possible-bug-in-stats/6461/8

changelog/unreleased/pull-4526

@@ -0,0 +1,11 @@
+Enhancement: Add bitrot detection to `diff` command
+
+The output of the `diff` command now includes the modifier `?` for files
+to indicate bitrot in backed up files. It will appear whenever there is a
+difference in content while the metadata is exactly the same. Since files with
+unchanged metadata are normally not read again when creating a backup, the
+detection is only effective if the right-hand side of the diff has been created
+with "backup --force".
+
+https://github.com/restic/restic/issues/805
+https://github.com/restic/restic/pull/4526

changelog/unreleased/pull-4573

@@ -0,0 +1,5 @@
+Enhancement: Add `--new-host` and `--new-time` options to `rewrite` command
+
+`restic rewrite` now allows rewriting the host and / or time metadata of a snapshot.
+
+https://github.com/restic/restic/pull/4573

changelog/unreleased/pull-4590

@@ -0,0 +1,7 @@
+Enhancement: `mount` tests mountpoint existence before opening the repository
+
+The restic `mount` command now checks for the existence of the
+mountpoint before opening the repository, leading to quicker error
+detection.
+
+https://github.com/restic/restic/pull/4590

changelog/unreleased/pull-4611

@@ -0,0 +1,7 @@
+Enhancement: Back up windows created time and file attributes like hidden flag
+
+Restic did not back up windows-specific meta-data like created time and file attributes like hidden flag.
+Restic now backs up file created time and file attributes like hidden, readonly and encrypted flag when backing up files and folders on Windows.
+
+https://github.com/restic/restic/pull/4611
+

changelog/unreleased/pull-4615

@@ -0,0 +1,6 @@
+Bugfix: `find` ignored directories in some cases
+
+In some cases, the `find` command ignored empty or moved directories. This has
+been fixed.
+
+https://github.com/restic/restic/pull/4615

changelog/unreleased/pull-4644

@@ -0,0 +1,10 @@
+Enhancement: Improve `repair packs` command
+
+The `repair packs` command has been improved to also be able to process
+truncated pack files. The `check --read-data` command will provide instructions
+on using the command if necessary to repair a repository. See the guide at
+https://restic.readthedocs.io/en/stable/077_troubleshooting.html for further
+instructions.
+
+https://github.com/restic/restic/pull/4644
+https://github.com/restic/restic/pull/4655

changelog/unreleased/pull-4664

@@ -0,0 +1,8 @@
+Enhancement: `ls` uses `message_type` field to distinguish JSON messages
+
+The `ls` command was the only command that used the `struct_type` field to determine
+the message type in the JSON output format. Now, the JSON output of the
+`ls` command also includes the `message_type`. The `struct_type` field is
+still included, but it deprecated.
+
+https://github.com/restic/restic/pull/4664

changelog/unreleased/pull-4703

@@ -0,0 +1,9 @@
+Bugfix: Shutdown cleanly when SIGTERM is received
+
+Prior, if restic received SIGTERM it'd just immediately terminate skipping
+cleanup- resulting in potential issues like stale locks being left behind.
+
+This primarily effected containerized restic invocations- they use SIGTERM-
+but this could be triggered via a simple `killall restic` in addition.
+
+https://github.com/restic/restic/pull/4703

changelog/unreleased/pull-4708

@@ -0,0 +1,11 @@
+Enhancement: Back up and restore SecurityDescriptors on Windows
+
+Restic now backs up and restores SecurityDescriptors when backing up files and folders 
+on Windows which includes owner, group, discretionary access control list (DACL), 
+system access control list (SACL). This requires the user to be a member of backup 
+operators or the application must be run as admin.
+If that is not the case, only the current user's owner, group and DACL will be backed up 
+and during restore only the DACL of the backed file will be restored while the current 
+user's owner and group will be set during the restore.
+
+https://github.com/restic/restic/pull/4708

changelog/unreleased/pull-4709

@@ -0,0 +1,10 @@
+Bugfix: Correct `--no-lock` handling of `ls` and `tag` command
+
+The `ls` command never locked the repository. This has been fixed. The old
+behavior is still supported using `ls --no-lock`. The latter invocation also
+works with older restic versions.
+
+The `tag` command erroneously accepted the `--no-lock` command. The command
+now always requires an exclusive lock.
+
+https://github.com/restic/restic/pull/4709

changelog/unreleased/pull-4737

@@ -0,0 +1,5 @@
+Enhancement: include snapshot id in reason field of forget JSON output
+
+The JSON output of the `forget` command now includes the `id` and `short_id` of a snapshot in the `reason` field.
+
+https://github.com/restic/restic/pull/4737

changelog/unreleased/pull-4796

@@ -0,0 +1,8 @@
+Enhancement: Improve `dump` performance for large files
+
+The `dump` command now retrieves the data chunks for a file in parallel. This
+improves the download performance by up to the configured number of parallel
+backend connections.
+
+https://github.com/restic/restic/issues/3406
+https://github.com/restic/restic/pull/4796

changelog/unreleased/pull-4799

@@ -0,0 +1,5 @@
+Enhancement: Add option to force use of Azure CLI credential
+
+A new environment variable `AZURE_FORCE_CLI_CREDENTIAL=true` allows forcing the use of Azure CLI credential, ignoring other credentials like managed identity.
+
+https://github.com/restic/restic/pull/4799

cmd/restic/cleanup.go

@@ -1,89 +1,41 @@
 package main
 
 import (
+	"context"
 	"os"
 	"os/signal"
-	"sync"
 	"syscall"
 
 	"github.com/restic/restic/internal/debug"
 )
 
-var cleanupHandlers struct {
-	sync.Mutex
-	list []func(code int) (int, error)
-	done bool
-	ch   chan os.Signal
+func createGlobalContext() context.Context {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	ch := make(chan os.Signal, 1)
+	go cleanupHandler(ch, cancel)
+	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
+
+	return ctx
 }
 
-func init() {
-	cleanupHandlers.ch = make(chan os.Signal, 1)
-	go CleanupHandler(cleanupHandlers.ch)
-	signal.Notify(cleanupHandlers.ch, syscall.SIGINT)
-}
+// cleanupHandler handles the SIGINT and SIGTERM signals.
+func cleanupHandler(c <-chan os.Signal, cancel context.CancelFunc) {
+	s := <-c
+	debug.Log("signal %v received, cleaning up", s)
+	Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s)
 
-// AddCleanupHandler adds the function f to the list of cleanup handlers so
-// that it is executed when all the cleanup handlers are run, e.g. when SIGINT
-// is received.
-func AddCleanupHandler(f func(code int) (int, error)) {
-	cleanupHandlers.Lock()
-	defer cleanupHandlers.Unlock()
-
-	// reset the done flag for integration tests
-	cleanupHandlers.done = false
-
-	cleanupHandlers.list = append(cleanupHandlers.list, f)
-}
-
-// RunCleanupHandlers runs all registered cleanup handlers
-func RunCleanupHandlers(code int) int {
-	cleanupHandlers.Lock()
-	defer cleanupHandlers.Unlock()
-
-	if cleanupHandlers.done {
-		return code
+	if val, _ := os.LookupEnv("RESTIC_DEBUG_STACKTRACE_SIGINT"); val != "" {
+		_, _ = os.Stderr.WriteString("\n--- STACKTRACE START ---\n\n")
+		_, _ = os.Stderr.WriteString(debug.DumpStacktrace())
+		_, _ = os.Stderr.WriteString("\n--- STACKTRACE END ---\n")
 	}
-	cleanupHandlers.done = true
 
-	for _, f := range cleanupHandlers.list {
-		var err error
-		code, err = f(code)
-		if err != nil {
-			Warnf("error in cleanup handler: %v\n", err)
-		}
-	}
-	cleanupHandlers.list = nil
-	return code
+	cancel()
 }
 
-// CleanupHandler handles the SIGINT signals.
-func CleanupHandler(c <-chan os.Signal) {
-	for s := range c {
-		debug.Log("signal %v received, cleaning up", s)
-		Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s)
-
-		if val, _ := os.LookupEnv("RESTIC_DEBUG_STACKTRACE_SIGINT"); val != "" {
-			_, _ = os.Stderr.WriteString("\n--- STACKTRACE START ---\n\n")
-			_, _ = os.Stderr.WriteString(debug.DumpStacktrace())
-			_, _ = os.Stderr.WriteString("\n--- STACKTRACE END ---\n")
-		}
-
-		code := 0
-
-		if s == syscall.SIGINT {
-			code = 130
-		} else {
-			code = 1
-		}
-
-		Exit(code)
-	}
-}
-
-// Exit runs the cleanup handlers and then terminates the process with the
-// given exit code.
+// Exit terminates the process with the given exit code.
 func Exit(code int) {
-	code = RunCleanupHandlers(code)
 	debug.Log("exiting with status code %d", code)
 	os.Exit(code)
 }

cmd/restic/cmd_backup.go

@@ -12,7 +12,6 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -25,7 +24,6 @@ import (
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/textfile"
-	"github.com/restic/restic/internal/ui"
 	"github.com/restic/restic/internal/ui/backup"
 	"github.com/restic/restic/internal/ui/termstatus"
 )
@@ -44,7 +42,7 @@ Exit status is 0 if the command was successful.
 Exit status is 1 if there was a fatal error (no snapshot created).
 Exit status is 3 if some source data could not be read (incomplete snapshot created).
 `,
-	PreRun: func(cmd *cobra.Command, args []string) {
+	PreRun: func(_ *cobra.Command, _ []string) {
 		if backupOptions.Host == "" {
 			hostname, err := os.Hostname()
 			if err != nil {
@@ -56,31 +54,9 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
 	},
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		ctx := cmd.Context()
-		var wg sync.WaitGroup
-		cancelCtx, cancel := context.WithCancel(ctx)
-		defer func() {
-			// shutdown termstatus
-			cancel()
-			wg.Wait()
-		}()
-
-		term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet)
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			term.Run(cancelCtx)
-		}()
-
-		// use the terminal for stdout/stderr
-		prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr
-		defer func() {
-			globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr
-		}()
-		stdioWrapper := ui.NewStdioWrapper(term)
-		globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr()
-
-		return runBackup(ctx, backupOptions, globalOptions, term, args)
+		term, cancel := setupTermstatus()
+		defer cancel()
+		return runBackup(cmd.Context(), backupOptions, globalOptions, term, args)
 	},
 }
 
@@ -97,6 +73,7 @@ type BackupOptions struct {
 	ExcludeLargerThan string
 	Stdin             bool
 	StdinFilename     string
+	StdinCommand      bool
 	Tags              restic.TagLists
 	Host              string
 	FilesFrom         []string
@@ -134,9 +111,10 @@ func init() {
 	f.StringVar(&backupOptions.ExcludeLargerThan, "exclude-larger-than", "", "max `size` of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)")
 	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
 	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin")
+	f.BoolVar(&backupOptions.StdinCommand, "stdin-from-command", false, "interpret arguments as command to execute and store its stdout")
 	f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)")
 	f.UintVar(&backupOptions.ReadConcurrency, "read-concurrency", 0, "read `n` files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)")
-	f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
+	f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually (default: $RESTIC_HOST). To prevent an expensive rescan use the \"parent\" flag")
 	f.StringVar(&backupOptions.Host, "hostname", "", "set the `hostname` for the snapshot manually")
 	err := f.MarkDeprecated("hostname", "use --host")
 	if err != nil {
@@ -148,7 +126,7 @@ func init() {
 	f.StringArrayVar(&backupOptions.FilesFromRaw, "files-from-raw", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
 	f.StringVar(&backupOptions.TimeStamp, "time", "", "`time` of the backup (ex. '2012-11-01 22:08:41') (default: now)")
 	f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
-	f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files")
+	f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number and ctime changes when checking for modified files")
 	f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
 	f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done")
 	f.BoolVar(&backupOptions.NoScan, "no-scan", false, "do not run scanner to estimate size of backup")
@@ -159,6 +137,11 @@ func init() {
 	// parse read concurrency from env, on error the default value will be used
 	readConcurrency, _ := strconv.ParseUint(os.Getenv("RESTIC_READ_CONCURRENCY"), 10, 32)
 	backupOptions.ReadConcurrency = uint(readConcurrency)
+
+	// parse host from env, if not exists or empty the default value will be used
+	if host := os.Getenv("RESTIC_HOST"); host != "" {
+		backupOptions.Host = host
+	}
 }
 
 // filterExisting returns a slice of all existing items, or an error if no
@@ -287,7 +270,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
 		}
 	}
 
-	if opts.Stdin {
+	if opts.Stdin || opts.StdinCommand {
 		if len(opts.FilesFrom) > 0 {
 			return errors.Fatal("--stdin and --files-from cannot be used together")
 		}
@@ -298,7 +281,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
 			return errors.Fatal("--stdin and --files-from-raw cannot be used together")
 		}
 
-		if len(args) > 0 {
+		if len(args) > 0 && !opts.StdinCommand {
 			return errors.Fatal("--stdin was specified and files/dirs were listed as arguments")
 		}
 	}
@@ -366,7 +349,7 @@ func collectRejectFuncs(opts BackupOptions, targets []string) (fs []RejectFunc,
 
 // collectTargets returns a list of target files/dirs from several sources.
 func collectTargets(opts BackupOptions, args []string) (targets []string, err error) {
-	if opts.Stdin {
+	if opts.Stdin || opts.StdinCommand {
 		return nil, nil
 	}
 
@@ -433,7 +416,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
 
 // parent returns the ID of the parent snapshot. If there is none, nil is
 // returned.
-func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) {
+func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) {
 	if opts.Force {
 		return nil, nil
 	}
@@ -453,7 +436,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
 		f.Tags = []restic.TagList{opts.Tags.Flatten()}
 	}
 
-	sn, _, err := f.FindLatest(ctx, repo.Backend(), repo, snName)
+	sn, _, err := f.FindLatest(ctx, repo, repo, snName)
 	// Snapshot not found is ok if no explicit parent was set
 	if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) {
 		err = nil
@@ -462,7 +445,16 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
 }
 
 func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
-	err := opts.Check(gopts, args)
+	var vsscfg fs.VSSConfig
+	var err error
+
+	if runtime.GOOS == "windows" {
+		if vsscfg, err = fs.ParseVSSConfig(gopts.extended); err != nil {
+			return err
+		}
+	}
+
+	err = opts.Check(gopts, args)
 	if err != nil {
 		return err
 	}
@@ -473,6 +465,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 	}
 
 	timeStamp := time.Now()
+	backupStart := timeStamp
 	if opts.TimeStamp != "" {
 		timeStamp, err = time.ParseInLocation(TimeFormat, opts.TimeStamp, time.Local)
 		if err != nil {
@@ -484,10 +477,11 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 		Verbosef("open repository\n")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, opts.DryRun)
 	if err != nil {
 		return err
 	}
+	defer unlock()
 
 	var progressPrinter backup.ProgressPrinter
 	if gopts.JSON {
@@ -499,22 +493,6 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 		calculateProgressInterval(!gopts.Quiet, gopts.JSON))
 	defer progressReporter.Done()
 
-	if opts.DryRun {
-		repo.SetDryRun()
-	}
-
-	if !gopts.JSON {
-		progressPrinter.V("lock repository")
-	}
-	if !opts.DryRun {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
-
 	// rejectByNameFuncs collect functions that can reject items from the backup based on path only
 	rejectByNameFuncs, err := collectRejectByNameFuncs(opts, repo)
 	if err != nil {
@@ -578,8 +556,8 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 			return err
 		}
 
-		errorHandler := func(item string, err error) error {
-			return progressReporter.Error(item, err)
+		errorHandler := func(item string, err error) {
+			_ = progressReporter.Error(item, err)
 		}
 
 		messageHandler := func(msg string, args ...interface{}) {
@@ -588,20 +566,28 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 			}
 		}
 
-		localVss := fs.NewLocalVss(errorHandler, messageHandler)
+		localVss := fs.NewLocalVss(errorHandler, messageHandler, vsscfg)
 		defer localVss.DeleteSnapshots()
 		targetFS = localVss
 	}
-	if opts.Stdin {
+
+	if opts.Stdin || opts.StdinCommand {
 		if !gopts.JSON {
 			progressPrinter.V("read data from stdin")
 		}
 		filename := path.Join("/", opts.StdinFilename)
+		var source io.ReadCloser = os.Stdin
+		if opts.StdinCommand {
+			source, err = fs.NewCommandReader(ctx, args, globalOptions.stderr)
+			if err != nil {
+				return err
+			}
+		}
 		targetFS = &fs.Reader{
 			ModTime:    timeStamp,
 			Name:       filename,
 			Mode:       0644,
-			ReadCloser: os.Stdin,
+			ReadCloser: source,
 		}
 		targets = []string{filename}
 	}
@@ -623,14 +609,20 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 		wg.Go(func() error { return sc.Scan(cancelCtx, targets) })
 	}
 
-	arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: backupOptions.ReadConcurrency})
+	arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: opts.ReadConcurrency})
 	arch.SelectByName = selectByNameFilter
 	arch.Select = selectFilter
 	arch.WithAtime = opts.WithAtime
 	success := true
 	arch.Error = func(item string, err error) error {
 		success = false
-		return progressReporter.Error(item, err)
+		reterr := progressReporter.Error(item, err)
+		// If we receive a fatal error during the execution of the snapshot,
+		// we abort the snapshot.
+		if reterr == nil && errors.IsFatal(err) {
+			reterr = err
+		}
+		return reterr
 	}
 	arch.CompleteItem = progressReporter.CompleteItem
 	arch.StartFile = progressReporter.StartFile
@@ -648,6 +640,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 	snapshotOpts := archiver.SnapshotOptions{
 		Excludes:       opts.Excludes,
 		Tags:           opts.Tags.Flatten(),
+		BackupStart:    backupStart,
 		Time:           timeStamp,
 		Hostname:       opts.Host,
 		ParentSnapshot: parentSnapshot,
@@ -657,7 +650,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 	if !gopts.JSON {
 		progressPrinter.V("start backup on %v", targets)
 	}
-	_, id, err := arch.Snapshot(ctx, targets, snapshotOpts)
+	_, id, summary, err := arch.Snapshot(ctx, targets, snapshotOpts)
 
 	// cleanly shutdown all running goroutines
 	cancel()
@@ -671,7 +664,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
 	}
 
 	// Report finished execution
-	progressReporter.Finish(id, opts.DryRun)
+	progressReporter.Finish(id, summary, opts.DryRun)
 	if !gopts.JSON && !opts.DryRun {
 		progressPrinter.P("snapshot %s saved\n", id.Str())
 	}

cmd/restic/cmd_backup_integration_test.go

@@ -249,29 +249,18 @@ func TestBackupTreeLoadError(t *testing.T) {
 	opts := BackupOptions{}
 	// Backup a subdirectory first, such that we can remove the tree pack for the subdirectory
 	testRunBackup(t, env.testdata, []string{"test"}, opts, env.gopts)
-
-	r, err := OpenRepository(context.TODO(), env.gopts)
-	rtest.OK(t, err)
-	rtest.OK(t, r.LoadIndex(context.TODO(), nil))
-	treePacks := restic.NewIDSet()
-	r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
-		if pb.Type == restic.TreeBlob {
-			treePacks.Insert(pb.PackID)
-		}
-	})
+	treePacks := listTreePacks(env.gopts, t)
 
 	testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
 	testRunCheck(t, env.gopts)
 
 	// delete the subdirectory pack first
-	for id := range treePacks {
-		rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}))
-	}
+	removePacks(env.gopts, t, treePacks)
 	testRunRebuildIndex(t, env.gopts)
 	// now the repo is missing the tree blob in the index; check should report this
 	testRunCheckMustFail(t, env.gopts)
 	// second backup should report an error but "heal" this situation
-	err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
+	err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
 	rtest.Assert(t, err != nil, "backup should have reported an error for the subdirectory")
 	testRunCheck(t, env.gopts)
 
@@ -405,6 +394,7 @@ func TestIncrementalBackup(t *testing.T) {
 	t.Logf("repository grown by %d bytes", stat3.size-stat2.size)
 }
 
+// nolint: staticcheck // false positive nil pointer dereference check
 func TestBackupTags(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
@@ -440,6 +430,7 @@ func TestBackupTags(t *testing.T) {
 		"expected parent to be %v, got %v", parent.ID, newest.Parent)
 }
 
+// nolint: staticcheck // false positive nil pointer dereference check
 func TestBackupProgramVersion(t *testing.T) {
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
@@ -567,3 +558,72 @@ func linkEqual(source, dest []string) bool {
 
 	return true
 }
+
+func TestStdinFromCommand(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{
+		StdinCommand:  true,
+		StdinFilename: "stdin",
+	}
+
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; print('something'); sys.exit(0)"}, opts, env.gopts)
+	testListSnapshots(t, env.gopts, 1)
+
+	testRunCheck(t, env.gopts)
+}
+
+func TestStdinFromCommandNoOutput(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{
+		StdinCommand:  true,
+		StdinFilename: "stdin",
+	}
+
+	err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; sys.exit(0)"}, opts, env.gopts)
+	rtest.Assert(t, err != nil && err.Error() == "at least one source file could not be read", "No data error expected")
+	testListSnapshots(t, env.gopts, 1)
+
+	testRunCheck(t, env.gopts)
+}
+
+func TestStdinFromCommandFailExitCode(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{
+		StdinCommand:  true,
+		StdinFilename: "stdin",
+	}
+
+	err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; print('test'); sys.exit(1)"}, opts, env.gopts)
+	rtest.Assert(t, err != nil, "Expected error while backing up")
+
+	testListSnapshots(t, env.gopts, 0)
+
+	testRunCheck(t, env.gopts)
+}
+
+func TestStdinFromCommandFailNoOutputAndExitCode(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testSetupBackupData(t, env)
+	opts := BackupOptions{
+		StdinCommand:  true,
+		StdinFilename: "stdin",
+	}
+
+	err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; sys.exit(1)"}, opts, env.gopts)
+	rtest.Assert(t, err != nil, "Expected error while backing up")
+
+	testListSnapshots(t, env.gopts, 0)
+
+	testRunCheck(t, env.gopts)
+}

cmd/restic/cmd_cache.go

@@ -28,7 +28,7 @@ EXIT STATUS
 Exit status is 0 if the command was successful, and non-zero if there was any error.
 `,
 	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
+	RunE: func(_ *cobra.Command, args []string) error {
 		return runCache(cacheOptions, globalOptions, args)
 	},
 }

cmd/restic/cmd_cat.go

@@ -7,7 +7,6 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
@@ -64,19 +63,11 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 		return err
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
-
-	if !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
+	defer unlock()
 
 	tpe := args[0]
 
@@ -106,7 +97,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 		Println(string(buf))
 		return nil
 	case "snapshot":
-		sn, _, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
+		sn, _, err := restic.FindSnapshot(ctx, repo, repo, args[1])
 		if err != nil {
 			return errors.Fatalf("could not find snapshot: %v\n", err)
 		}
@@ -154,9 +145,9 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 		return nil
 
 	case "pack":
-		h := restic.Handle{Type: restic.PackFile, Name: id.String()}
-		buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
-		if err != nil {
+		buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
+		// allow returning broken pack files
+		if buf == nil {
 			return err
 		}
 
@@ -193,7 +184,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
 		return errors.Fatal("blob not found")
 
 	case "tree":
-		sn, subfolder, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
+		sn, subfolder, err := restic.FindSnapshot(ctx, repo, repo, args[1])
 		if err != nil {
 			return errors.Fatalf("could not find snapshot: %v\n", err)
 		}

cmd/restic/cmd_check.go

@@ -15,6 +15,7 @@ import (
 	"github.com/restic/restic/internal/checker"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
+	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/ui"
 )
@@ -38,7 +39,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return runCheck(cmd.Context(), checkOptions, globalOptions, args)
 	},
-	PreRunE: func(cmd *cobra.Command, args []string) error {
+	PreRunE: func(_ *cobra.Command, _ []string) error {
 		return checkFlags(checkOptions)
 	},
 }
@@ -199,25 +200,16 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	}
 
 	cleanup := prepareCheckCache(opts, &gopts)
-	AddCleanupHandler(func(code int) (int, error) {
-		cleanup()
-		return code, nil
-	})
-
-	repo, err := OpenRepository(ctx, gopts)
-	if err != nil {
-		return err
-	}
+	defer cleanup()
 
 	if !gopts.NoLock {
 		Verbosef("create exclusive lock for repository\n")
-		var lock *restic.Lock
-		lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
 	}
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, gopts.NoLock)
+	if err != nil {
+		return err
+	}
+	defer unlock()
 
 	chkr := checker.New(repo, opts.CheckUnused)
 	err = chkr.LoadSnapshots(ctx)
@@ -228,15 +220,23 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	Verbosef("load indexes\n")
 	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
 	hints, errs := chkr.LoadIndex(ctx, bar)
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 
 	errorsFound := false
 	suggestIndexRebuild := false
+	suggestLegacyIndexRebuild := false
 	mixedFound := false
 	for _, hint := range hints {
 		switch hint.(type) {
-		case *checker.ErrDuplicatePacks, *checker.ErrOldIndexFormat:
+		case *checker.ErrDuplicatePacks:
 			Printf("%v\n", hint)
 			suggestIndexRebuild = true
+		case *checker.ErrOldIndexFormat:
+			Warnf("error: %v\n", hint)
+			suggestLegacyIndexRebuild = true
+			errorsFound = true
 		case *checker.ErrMixedPack:
 			Printf("%v\n", hint)
 			mixedFound = true
@@ -247,7 +247,10 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	}
 
 	if suggestIndexRebuild {
-		Printf("Duplicate packs/old indexes are non-critical, you can run `restic repair index' to correct this.\n")
+		Printf("Duplicate packs are non-critical, you can run `restic repair index' to correct this.\n")
+	}
+	if suggestLegacyIndexRebuild {
+		Warnf("Found indexes using the legacy format, you must run `restic repair index' to correct this.\n")
 	}
 	if mixedFound {
 		Printf("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
@@ -281,6 +284,9 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	if orphanedPacks > 0 {
 		Verbosef("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks)
 	}
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 
 	Verbosef("check snapshots, trees and blobs\n")
 	errChan = make(chan error)
@@ -314,9 +320,16 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 	// Must happen after `errChan` is read from in the above loop to avoid
 	// deadlocking in the case of errors.
 	wg.Wait()
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 
 	if opts.CheckUnused {
-		for _, id := range chkr.UnusedBlobs(ctx) {
+		unused, err := chkr.UnusedBlobs(ctx)
+		if err != nil {
+			return err
+		}
+		for _, id := range unused {
 			Verbosef("unused blob %v\n", id)
 			errorsFound = true
 		}
@@ -335,21 +348,19 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 		for err := range errChan {
 			errorsFound = true
 			Warnf("%v\n", err)
-			if err, ok := err.(*checker.ErrPackData); ok {
-				if strings.Contains(err.Error(), "wrong data returned, hash is") {
-					salvagePacks = append(salvagePacks, err.PackID)
-				}
+			if err, ok := err.(*repository.ErrPackData); ok {
+				salvagePacks = append(salvagePacks, err.PackID)
 			}
 		}
 		p.Done()
 
 		if len(salvagePacks) > 0 {
-			Warnf("\nThe repository contains pack files with damaged blobs. These blobs must be removed to repair the repository. This can be done using the following commands:\n\n")
-			var strIds []string
+			Warnf("\nThe repository contains pack files with damaged blobs. These blobs must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n")
+			var strIDs []string
 			for _, id := range salvagePacks {
-				strIds = append(strIds, id.String())
+				strIDs = append(strIDs, id.String())
 			}
-			Warnf("RESTIC_FEATURES=repair-packs-v1 restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIds, " "))
+			Warnf("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIDs, " "))
 			Warnf("Corrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n")
 		}
 	}
@@ -395,10 +406,13 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
 		doReadData(packs)
 	}
 
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+
 	if errorsFound {
 		return errors.Fatal("repository contains errors")
 	}
-
 	Verbosef("no errors were found\n")
 
 	return nil
@@ -417,7 +431,7 @@ func selectPacksByBucket(allPacks map[restic.ID]int64, bucket, totalBuckets uint
 	return packs
 }
 
-// selectRandomPacksByPercentage selects the given percentage of packs which are randomly choosen.
+// selectRandomPacksByPercentage selects the given percentage of packs which are randomly chosen.
 func selectRandomPacksByPercentage(allPacks map[restic.ID]int64, percentage float64) map[restic.ID]int64 {
 	packCount := len(allPacks)
 	packsToCheck := int(float64(packCount) * (percentage / 100.0))

cmd/restic/cmd_check_test.go

@@ -71,7 +71,7 @@ func TestSelectPacksByBucket(t *testing.T) {
 	var testPacks = make(map[restic.ID]int64)
 	for i := 1; i <= 10; i++ {
 		id := restic.NewRandomID()
-		// ensure relevant part of generated id is reproducable
+		// ensure relevant part of generated id is reproducible
 		id[0] = byte(i)
 		testPacks[id] = 0
 	}
@@ -124,7 +124,7 @@ func TestSelectRandomPacksByPercentage(t *testing.T) {
 }
 
 func TestSelectNoRandomPacksByPercentage(t *testing.T) {
-	// that the a repository without pack files works
+	// that the repository without pack files works
 	var testPacks = make(map[restic.ID]int64)
 	selectedPacks := selectRandomPacksByPercentage(testPacks, 10.0)
 	rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
@@ -158,7 +158,7 @@ func TestSelectRandomPacksByFileSize(t *testing.T) {
 }
 
 func TestSelectNoRandomPacksByFileSize(t *testing.T) {
-	// that the a repository without pack files works
+	// that the repository without pack files works
 	var testPacks = make(map[restic.ID]int64)
 	selectedPacks := selectRandomPacksByFileSize(testPacks, 10, 500)
 	rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")

cmd/restic/cmd_copy.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
@@ -54,7 +53,7 @@ func init() {
 }
 
 func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error {
-	secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination")
+	secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(ctx, opts.secondaryRepoOptions, gopts, "destination")
 	if err != nil {
 		return err
 	}
@@ -63,37 +62,24 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
 		gopts, secondaryGopts = secondaryGopts, gopts
 	}
 
-	srcRepo, err := OpenRepository(ctx, gopts)
+	ctx, srcRepo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
+	if err != nil {
+		return err
+	}
+	defer unlock()
+
+	ctx, dstRepo, unlock, err := openWithAppendLock(ctx, secondaryGopts, false)
+	if err != nil {
+		return err
+	}
+	defer unlock()
+
+	srcSnapshotLister, err := restic.MemorizeList(ctx, srcRepo, restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
 
-	dstRepo, err := OpenRepository(ctx, secondaryGopts)
-	if err != nil {
-		return err
-	}
-
-	if !gopts.NoLock {
-		var srcLock *restic.Lock
-		srcLock, ctx, err = lockRepo(ctx, srcRepo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(srcLock)
-		if err != nil {
-			return err
-		}
-	}
-
-	dstLock, ctx, err := lockRepo(ctx, dstRepo, gopts.RetryLock, gopts.JSON)
-	defer unlockRepo(dstLock)
-	if err != nil {
-		return err
-	}
-
-	srcSnapshotLister, err := backend.MemorizeList(ctx, srcRepo.Backend(), restic.SnapshotFile)
-	if err != nil {
-		return err
-	}
-
-	dstSnapshotLister, err := backend.MemorizeList(ctx, dstRepo.Backend(), restic.SnapshotFile)
+	dstSnapshotLister, err := restic.MemorizeList(ctx, dstRepo, restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
@@ -117,6 +103,9 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
 		// also consider identical snapshot copies
 		dstSnapshotByOriginal[*sn.ID()] = append(dstSnapshotByOriginal[*sn.ID()], sn)
 	}
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 
 	// remember already processed trees across all snapshots
 	visitedTrees := restic.NewIDSet()
@@ -127,11 +116,12 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
 		if sn.Original != nil {
 			srcOriginal = *sn.Original
 		}
+
 		if originalSns, ok := dstSnapshotByOriginal[srcOriginal]; ok {
 			isCopy := false
 			for _, originalSn := range originalSns {
 				if similarSnapshots(originalSn, sn) {
-					Verboseff("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
+					Verboseff("\n%v\n", sn)
 					Verboseff("skipping source snapshot %s, was already copied to snapshot %s\n", sn.ID().Str(), originalSn.ID().Str())
 					isCopy = true
 					break
@@ -141,7 +131,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
 				continue
 			}
 		}
-		Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
+		Verbosef("\n%v\n", sn)
 		Verbosef("  copy started, this may take a while...\n")
 		if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, gopts.Quiet); err != nil {
 			return err
@@ -160,7 +150,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
 		}
 		Verbosef("snapshot %s saved\n", newID.Str())
 	}
-	return nil
+	return ctx.Err()
 }
 
 func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool {

cmd/restic/cmd_debug.go

@@ -20,7 +20,6 @@ import (
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/errgroup"
 
-	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/crypto"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/index"
@@ -52,19 +51,23 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 	},
 }
 
-var tryRepair bool
-var repairByte bool
-var extractPack bool
-var reuploadBlobs bool
+type DebugExamineOptions struct {
+	TryRepair     bool
+	RepairByte    bool
+	ExtractPack   bool
+	ReuploadBlobs bool
+}
+
+var debugExamineOpts DebugExamineOptions
 
 func init() {
 	cmdRoot.AddCommand(cmdDebug)
 	cmdDebug.AddCommand(cmdDebugDump)
 	cmdDebug.AddCommand(cmdDebugExamine)
-	cmdDebugExamine.Flags().BoolVar(&extractPack, "extract-pack", false, "write blobs to the current directory")
-	cmdDebugExamine.Flags().BoolVar(&reuploadBlobs, "reupload-blobs", false, "reupload blobs to the repository")
-	cmdDebugExamine.Flags().BoolVar(&tryRepair, "try-repair", false, "try to repair broken blobs with single bit flips")
-	cmdDebugExamine.Flags().BoolVar(&repairByte, "repair-byte", false, "try to repair broken blobs by trying bytes")
+	cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.ExtractPack, "extract-pack", false, "write blobs to the current directory")
+	cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.ReuploadBlobs, "reupload-blobs", false, "reupload blobs to the repository")
+	cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.TryRepair, "try-repair", false, "try to repair broken blobs with single bit flips")
+	cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.RepairByte, "repair-byte", false, "try to repair broken blobs by trying bytes")
 }
 
 func prettyPrintJSON(wr io.Writer, item interface{}) error {
@@ -78,7 +81,7 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error {
 }
 
 func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
-	return restic.ForAllSnapshots(ctx, repo.Backend(), repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
+	return restic.ForAllSnapshots(ctx, repo, repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
 		if err != nil {
 			return err
 		}
@@ -107,7 +110,7 @@ type Blob struct {
 func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
 
 	var m sync.Mutex
-	return restic.ParallelList(ctx, repo.Backend(), restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
+	return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
 		blobs, _, err := repo.ListPack(ctx, id, size)
 		if err != nil {
 			Warnf("error for pack %v: %v\n", id.Str(), err)
@@ -133,8 +136,8 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
 	})
 }
 
-func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error {
-	return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
+func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer) error {
+	return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
 		Printf("index_id: %v\n", id)
 		if err != nil {
 			return err
@@ -149,19 +152,11 @@ func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error
 		return errors.Fatal("type not specified")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
-
-	if !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
+	defer unlock()
 
 	tpe := args[0]
 
@@ -196,7 +191,7 @@ var cmdDebugExamine = &cobra.Command{
 	Short:             "Examine a pack file",
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runDebugExamine(cmd.Context(), globalOptions, args)
+		return runDebugExamine(cmd.Context(), globalOptions, debugExamineOpts, args)
 	},
 }
 
@@ -290,7 +285,7 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by
 	})
 	err := wg.Wait()
 	if err != nil {
-		panic("all go rountines can only return nil")
+		panic("all go routines can only return nil")
 	}
 
 	if !found {
@@ -315,39 +310,32 @@ func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte {
 	return out
 }
 
-func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, list []restic.Blob) error {
+func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, packID restic.ID, list []restic.Blob) error {
 	dec, err := zstd.NewReader(nil)
 	if err != nil {
 		panic(err)
 	}
-	be := repo.Backend()
-	h := restic.Handle{
-		Name: packID.String(),
-		Type: restic.PackFile,
+
+	pack, err := repo.LoadRaw(ctx, restic.PackFile, packID)
+	// allow processing broken pack files
+	if pack == nil {
+		return err
 	}
 
 	wg, ctx := errgroup.WithContext(ctx)
 
-	if reuploadBlobs {
+	if opts.ReuploadBlobs {
 		repo.StartPackUploader(ctx, wg)
 	}
 
 	wg.Go(func() error {
 		for _, blob := range list {
 			Printf("      loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length)
-			buf := make([]byte, blob.Length)
-			err := be.Load(ctx, h, int(blob.Length), int64(blob.Offset), func(rd io.Reader) error {
-				n, err := io.ReadFull(rd, buf)
-				if err != nil {
-					return fmt.Errorf("read error after %d bytes: %v", n, err)
-				}
-				return nil
-			})
-			if err != nil {
-				Warnf("error read: %v\n", err)
+			if int(blob.Offset+blob.Length) > len(pack) {
+				Warnf("skipping truncated blob\n")
 				continue
 			}
-
+			buf := pack[blob.Offset : blob.Offset+blob.Length]
 			key := repo.Key()
 
 			nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
@@ -356,8 +344,8 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li
 			filePrefix := ""
 			if err != nil {
 				Warnf("error decrypting blob: %v\n", err)
-				if tryRepair || repairByte {
-					plaintext = tryRepairWithBitflip(ctx, key, buf, repairByte)
+				if opts.TryRepair || opts.RepairByte {
+					plaintext = tryRepairWithBitflip(ctx, key, buf, opts.RepairByte)
 				}
 				if plaintext != nil {
 					outputPrefix = "repaired "
@@ -391,13 +379,13 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li
 				Printf("         successfully %vdecrypted blob (length %v), hash is %v, ID matches\n", outputPrefix, len(plaintext), id)
 				prefix = "correct-"
 			}
-			if extractPack {
+			if opts.ExtractPack {
 				err = storePlainBlob(id, filePrefix+prefix, plaintext)
 				if err != nil {
 					return err
 				}
 			}
-			if reuploadBlobs {
+			if opts.ReuploadBlobs {
 				_, _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true)
 				if err != nil {
 					return err
@@ -406,7 +394,7 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li
 			}
 		}
 
-		if reuploadBlobs {
+		if opts.ReuploadBlobs {
 			return repo.Flush(ctx)
 		}
 		return nil
@@ -437,17 +425,22 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
 	return nil
 }
 
-func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) error {
-	repo, err := OpenRepository(ctx, gopts)
+func runDebugExamine(ctx context.Context, gopts GlobalOptions, opts DebugExamineOptions, args []string) error {
+	if opts.ExtractPack && gopts.NoLock {
+		return fmt.Errorf("--extract-pack and --no-lock are mutually exclusive")
+	}
+
+	ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
+	defer unlock()
 
 	ids := make([]restic.ID, 0)
 	for _, name := range args {
 		id, err := restic.ParseID(name)
 		if err != nil {
-			id, err = restic.Find(ctx, repo.Backend(), restic.PackFile, name)
+			id, err = restic.Find(ctx, repo, restic.PackFile, name)
 			if err != nil {
 				Warnf("error: %v\n", err)
 				continue
@@ -460,15 +453,6 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er
 		return errors.Fatal("no pack files to examine")
 	}
 
-	if !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
-
 	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
 	err = repo.LoadIndex(ctx, bar)
 	if err != nil {
@@ -476,7 +460,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er
 	}
 
 	for _, id := range ids {
-		err := examinePack(ctx, repo, id)
+		err := examinePack(ctx, opts, repo, id)
 		if err != nil {
 			Warnf("error: %v\n", err)
 		}
@@ -487,23 +471,15 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er
 	return nil
 }
 
-func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) error {
+func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID) error {
 	Printf("examine %v\n", id)
 
-	h := restic.Handle{
-		Type: restic.PackFile,
-		Name: id.String(),
-	}
-	fi, err := repo.Backend().Stat(ctx, h)
-	if err != nil {
-		return err
-	}
-	Printf("  file size is %v\n", fi.Size)
-
-	buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
-	if err != nil {
+	buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
+	// also process damaged pack files
+	if buf == nil {
 		return err
 	}
+	Printf("  file size is %v\n", len(buf))
 	gotID := restic.Hash(buf)
 	if !id.Equal(gotID) {
 		Printf("  wanted hash %v, got %v\n", id, gotID)
@@ -522,9 +498,9 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro
 			continue
 		}
 
-		checkPackSize(blobs, fi.Size)
+		checkPackSize(blobs, len(buf))
 
-		err = loadBlobs(ctx, repo, id, blobs)
+		err = loadBlobs(ctx, opts, repo, id, blobs)
 		if err != nil {
 			Warnf("error: %v\n", err)
 		} else {
@@ -535,19 +511,19 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro
 	Printf("  ========================================\n")
 	Printf("  inspect the pack itself\n")
 
-	blobs, _, err := repo.ListPack(ctx, id, fi.Size)
+	blobs, _, err := repo.ListPack(ctx, id, int64(len(buf)))
 	if err != nil {
 		return fmt.Errorf("pack %v: %v", id.Str(), err)
 	}
-	checkPackSize(blobs, fi.Size)
+	checkPackSize(blobs, len(buf))
 
 	if !blobsLoaded {
-		return loadBlobs(ctx, repo, id, blobs)
+		return loadBlobs(ctx, opts, repo, id, blobs)
 	}
 	return nil
 }
 
-func checkPackSize(blobs []restic.Blob, fileSize int64) {
+func checkPackSize(blobs []restic.Blob, fileSize int) {
 	// track current size and offset
 	var size, offset uint64
 

cmd/restic/cmd_diff.go

@@ -7,7 +7,6 @@ import (
 	"reflect"
 	"sort"
 
-	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
@@ -28,6 +27,10 @@ directory:
 * U  The metadata (access mode, timestamps, ...) for the item was updated
 * M  The file's content was modified
 * T  The type was changed, e.g. a file was made a symlink
+* ?  Bitrot detected: The file's content has changed but all metadata is the same
+
+Metadata comparison will likely not work if a backup was created using the
+'--ignore-inode' or '--ignore-ctime' option.
 
 To only compare files in specific subfolders, you can use the
 "<snapshotID>:<subfolder>" syntax, where "subfolder" is a path within the
@@ -58,7 +61,7 @@ func init() {
 	f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata")
 }
 
-func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) {
+func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.LoaderUnpacked, desc string) (*restic.Snapshot, string, error) {
 	sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc)
 	if err != nil {
 		return nil, "", errors.Fatal(err.Error())
@@ -68,7 +71,7 @@ func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository,
 
 // Comparer collects all things needed to compare two snapshots.
 type Comparer struct {
-	repo        restic.Repository
+	repo        restic.BlobLoader
 	opts        DiffOptions
 	printChange func(change *Change)
 }
@@ -144,7 +147,7 @@ type DiffStatsContainer struct {
 }
 
 // updateBlobs updates the blob counters in the stats struct.
-func updateBlobs(repo restic.Repository, blobs restic.BlobSet, stats *DiffStat) {
+func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat) {
 	for h := range blobs {
 		switch h.Type {
 		case restic.DataBlob:
@@ -273,6 +276,16 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
 				!reflect.DeepEqual(node1.Content, node2.Content) {
 				mod += "M"
 				stats.ChangedFiles++
+
+				node1NilContent := *node1
+				node2NilContent := *node2
+				node1NilContent.Content = nil
+				node2NilContent.Content = nil
+				// the bitrot detection may not work if `backup --ignore-inode` or `--ignore-ctime` were used
+				if node1NilContent.Equals(node2NilContent) {
+					// probable bitrot detected
+					mod += "?"
+				}
 			} else if c.opts.ShowMetadata && !node1.Equals(*node2) {
 				mod += "U"
 			}
@@ -331,22 +344,14 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 		return errors.Fatalf("specify two snapshot IDs")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
-
-	if !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
+	defer unlock()
 
 	// cache snapshots listing
-	be, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
+	be, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
@@ -388,7 +393,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 
 	c := &Comparer{
 		repo: repo,
-		opts: diffOptions,
+		opts: opts,
 		printChange: func(change *Change) {
 			Printf("%-5s%v\n", change.Modifier, change.Path)
 		},
@@ -405,7 +410,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
 	}
 
 	if gopts.Quiet {
-		c.printChange = func(change *Change) {}
+		c.printChange = func(_ *Change) {}
 	}
 
 	stats := &DiffStatsContainer{

cmd/restic/cmd_dump.go

@@ -46,6 +46,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 type DumpOptions struct {
 	restic.SnapshotFilter
 	Archive string
+	Target  string
 }
 
 var dumpOptions DumpOptions
@@ -56,6 +57,7 @@ func init() {
 	flags := cmdDump.Flags()
 	initSingleSnapshotFilter(flags, &dumpOptions.SnapshotFilter)
 	flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"")
+	flags.StringVarP(&dumpOptions.Target, "target", "t", "", "write the output to target `path`")
 }
 
 func splitPath(p string) []string {
@@ -67,11 +69,11 @@ func splitPath(p string) []string {
 	return append(s, f)
 }
 
-func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, d *dump.Dumper) error {
+func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, canWriteArchiveFunc func() error) error {
 	// If we print / we need to assume that there are multiple nodes at that
 	// level in the tree.
 	if pathComponents[0] == "" {
-		if err := checkStdoutArchive(); err != nil {
+		if err := canWriteArchiveFunc(); err != nil {
 			return err
 		}
 		return d.DumpTree(ctx, tree, "/")
@@ -91,9 +93,9 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
 				if err != nil {
 					return errors.Wrapf(err, "cannot load subtree for %q", item)
 				}
-				return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d)
+				return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d, canWriteArchiveFunc)
 			case dump.IsDir(node):
-				if err := checkStdoutArchive(); err != nil {
+				if err := canWriteArchiveFunc(); err != nil {
 					return err
 				}
 				subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
@@ -129,25 +131,17 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
 
 	splittedPath := splitPath(path.Clean(pathToPrint))
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
-
-	if !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
+	defer unlock()
 
 	sn, subfolder, err := (&restic.SnapshotFilter{
 		Hosts: opts.Hosts,
 		Paths: opts.Paths,
 		Tags:  opts.Tags,
-	}).FindLatest(ctx, repo.Backend(), repo, snapshotIDString)
+	}).FindLatest(ctx, repo, repo, snapshotIDString)
 	if err != nil {
 		return errors.Fatalf("failed to find snapshot: %v", err)
 	}
@@ -168,8 +162,24 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
 		return errors.Fatalf("loading tree for snapshot %q failed: %v", snapshotIDString, err)
 	}
 
-	d := dump.New(opts.Archive, repo, os.Stdout)
-	err = printFromTree(ctx, tree, repo, "/", splittedPath, d)
+	outputFileWriter := os.Stdout
+	canWriteArchiveFunc := checkStdoutArchive
+
+	if opts.Target != "" {
+		file, err := os.Create(opts.Target)
+		if err != nil {
+			return fmt.Errorf("cannot dump to file: %w", err)
+		}
+		defer func() {
+			_ = file.Close()
+		}()
+
+		outputFileWriter = file
+		canWriteArchiveFunc = func() error { return nil }
+	}
+
+	d := dump.New(opts.Archive, repo, outputFileWriter)
+	err = printFromTree(ctx, tree, repo, "/", splittedPath, d, canWriteArchiveFunc)
 	if err != nil {
 		return errors.Fatalf("cannot dump file: %v", err)
 	}

cmd/restic/cmd_features.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/feature"
+	"github.com/restic/restic/internal/ui/table"
+
+	"github.com/spf13/cobra"
+)
+
+var featuresCmd = &cobra.Command{
+	Use:   "features",
+	Short: "Print list of feature flags",
+	Long: `
+The "features" command prints a list of supported feature flags.
+
+To pass feature flags to restic, set the RESTIC_FEATURES environment variable
+to "featureA=true,featureB=false". Specifying an unknown feature flag is an error.
+
+A feature can either be in alpha, beta, stable or deprecated state.
+An _alpha_ feature is disabled by default and may change in arbitrary ways between restic versions or be removed.
+A _beta_ feature is enabled by default, but still can change in minor ways or be removed.
+A _stable_ feature is always enabled and cannot be disabled. The flag will be removed in a future restic version.
+A _deprecated_ feature is always disabled and cannot be enabled. The flag will be removed in a future restic version.
+
+EXIT STATUS
+===========
+
+Exit status is 0 if the command was successful, and non-zero if there was any error.
+`,
+	Hidden:            true,
+	DisableAutoGenTag: true,
+	RunE: func(_ *cobra.Command, args []string) error {
+		if len(args) != 0 {
+			return errors.Fatal("the feature command expects no arguments")
+		}
+
+		fmt.Printf("All Feature Flags:\n")
+		flags := feature.Flag.List()
+
+		tab := table.New()
+		tab.AddColumn("Name", "{{ .Name }}")
+		tab.AddColumn("Type", "{{ .Type }}")
+		tab.AddColumn("Default", "{{ .Default }}")
+		tab.AddColumn("Description", "{{ .Description }}")
+
+		for _, flag := range flags {
+			tab.AddRow(flag)
+		}
+		return tab.Write(globalOptions.stdout)
+	},
+}
+
+func init() {
+	cmdRoot.AddCommand(featuresCmd)
+}

cmd/restic/cmd_find.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/filter"
@@ -127,6 +126,7 @@ func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) {
 		// Make the following attributes disappear
 		Name               byte `json:"name,omitempty"`
 		ExtendedAttributes byte `json:"extended_attributes,omitempty"`
+		GenericAttributes  byte `json:"generic_attributes,omitempty"`
 		Device             byte `json:"device,omitempty"`
 		Content            byte `json:"content,omitempty"`
 		Subtree            byte `json:"subtree,omitempty"`
@@ -245,13 +245,12 @@ func (s *statefulOutput) Finish() {
 
 // Finder bundles information needed to find a file or directory.
 type Finder struct {
-	repo        restic.Repository
-	pat         findPattern
-	out         statefulOutput
-	ignoreTrees restic.IDSet
-	blobIDs     map[string]struct{}
-	treeIDs     map[string]struct{}
-	itemsFound  int
+	repo       restic.Repository
+	pat        findPattern
+	out        statefulOutput
+	blobIDs    map[string]struct{}
+	treeIDs    map[string]struct{}
+	itemsFound int
 }
 
 func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error {
@@ -262,17 +261,17 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 	}
 
 	f.out.newsn = sn
-	return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
+	return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
 		if err != nil {
 			debug.Log("Error loading tree %v: %v", parentTreeID, err)
 
 			Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
 
-			return false, walker.ErrSkipNode
+			return walker.ErrSkipNode
 		}
 
 		if node == nil {
-			return false, nil
+			return nil
 		}
 
 		normalizedNodepath := nodepath
@@ -285,7 +284,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 		for _, pat := range f.pat.pattern {
 			found, err := filter.Match(pat, normalizedNodepath)
 			if err != nil {
-				return false, err
+				return err
 			}
 			if found {
 				foundMatch = true
@@ -293,16 +292,13 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 			}
 		}
 
-		var (
-			ignoreIfNoMatch = true
-			errIfNoMatch    error
-		)
+		var errIfNoMatch error
 		if node.Type == "dir" {
 			var childMayMatch bool
 			for _, pat := range f.pat.pattern {
 				mayMatch, err := filter.ChildMatch(pat, normalizedNodepath)
 				if err != nil {
-					return false, err
+					return err
 				}
 				if mayMatch {
 					childMayMatch = true
@@ -311,31 +307,28 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
 			}
 
 			if !childMayMatch {
-				ignoreIfNoMatch = true
 				errIfNoMatch = walker.ErrSkipNode
-			} else {
-				ignoreIfNoMatch = false
 			}
 		}
 
 		if !foundMatch {
-			return ignoreIfNoMatch, errIfNoMatch
+			return errIfNoMatch
 		}
 
 		if !f.pat.oldest.IsZero() && node.ModTime.Before(f.pat.oldest) {
 			debug.Log("    ModTime is older than %s\n", f.pat.oldest)
-			return ignoreIfNoMatch, errIfNoMatch
+			return errIfNoMatch
 		}
 
 		if !f.pat.newest.IsZero() && node.ModTime.After(f.pat.newest) {
 			debug.Log("    ModTime is newer than %s\n", f.pat.newest)
-			return ignoreIfNoMatch, errIfNoMatch
+			return errIfNoMatch
 		}
 
 		debug.Log("    found match\n")
 		f.out.PrintPattern(nodepath, node)
-		return false, nil
-	})
+		return nil
+	}})
 }
 
 func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
@@ -346,17 +339,17 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 	}
 
 	f.out.newsn = sn
-	return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
+	return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
 		if err != nil {
 			debug.Log("Error loading tree %v: %v", parentTreeID, err)
 
 			Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
 
-			return false, walker.ErrSkipNode
+			return walker.ErrSkipNode
 		}
 
 		if node == nil {
-			return false, nil
+			return nil
 		}
 
 		if node.Type == "dir" && f.treeIDs != nil {
@@ -374,7 +367,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 				// looking for blobs)
 				if f.itemsFound >= len(f.treeIDs) && f.blobIDs == nil {
 					// Return an error to terminate the Walk
-					return true, errors.New("OK")
+					return errors.New("OK")
 				}
 			}
 		}
@@ -395,8 +388,8 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
 			}
 		}
 
-		return false, nil
-	})
+		return nil
+	}})
 }
 
 var errAllPacksFound = errors.New("all packs found")
@@ -446,7 +439,10 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
 
 	if err != errAllPacksFound {
 		// try to resolve unknown pack ids from the index
-		packIDs = f.indexPacksToBlobs(ctx, packIDs)
+		packIDs, err = f.indexPacksToBlobs(ctx, packIDs)
+		if err != nil {
+			return err
+		}
 	}
 
 	if len(packIDs) > 0 {
@@ -463,13 +459,13 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
 	return nil
 }
 
-func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struct{}) map[string]struct{} {
+func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struct{}) (map[string]struct{}, error) {
 	wctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
 	// remember which packs were found in the index
 	indexPackIDs := make(map[string]struct{})
-	f.repo.Index().Each(wctx, func(pb restic.PackedBlob) {
+	err := f.repo.Index().Each(wctx, func(pb restic.PackedBlob) {
 		idStr := pb.PackID.String()
 		// keep entry in packIDs as Each() returns individual index entries
 		matchingID := false
@@ -488,6 +484,9 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
 			indexPackIDs[idStr] = struct{}{}
 		}
 	})
+	if err != nil {
+		return nil, err
+	}
 
 	for id := range indexPackIDs {
 		delete(packIDs, id)
@@ -500,7 +499,7 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
 		}
 		Warnf("some pack files are missing from the repository, getting their blobs from the repository index: %v\n\n", list)
 	}
-	return packIDs
+	return packIDs, nil
 }
 
 func (f *Finder) findObjectPack(id string, t restic.BlobType) {
@@ -570,21 +569,13 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
 		return errors.Fatal("cannot have several ID types")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
+	defer unlock()
 
-	if !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
-
-	snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
+	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
@@ -594,10 +585,9 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
 	}
 
 	f := &Finder{
-		repo:        repo,
-		pat:         pat,
-		out:         statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON},
-		ignoreTrees: restic.NewIDSet(),
+		repo: repo,
+		pat:  pat,
+		out:  statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON},
 	}
 
 	if opts.BlobID {
@@ -624,6 +614,9 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
 	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, opts.Snapshots) {
 		filteredSnapshots = append(filteredSnapshots, sn)
 	}
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 
 	sort.Slice(filteredSnapshots, func(i, j int) bool {
 		return filteredSnapshots[i].Time.Before(filteredSnapshots[j].Time)

cmd/restic/cmd_forget.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui/termstatus"
 	"github.com/spf13/cobra"
 )
 
@@ -33,7 +34,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runForget(cmd.Context(), forgetOptions, globalOptions, args)
+		term, cancel := setupTermstatus()
+		defer cancel()
+		return runForget(cmd.Context(), forgetOptions, forgetPruneOptions, globalOptions, term, args)
 	},
 }
 
@@ -98,6 +101,7 @@ type ForgetOptions struct {
 }
 
 var forgetOptions ForgetOptions
+var forgetPruneOptions PruneOptions
 
 func init() {
 	cmdRoot.AddCommand(cmdForget)
@@ -132,7 +136,7 @@ func init() {
 	f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
 
 	f.SortFlags = false
-	addPruneOptions(cmdForget)
+	addPruneOptions(cmdForget, &forgetPruneOptions)
 }
 
 func verifyForgetOptions(opts *ForgetOptions) error {
@@ -151,7 +155,7 @@ func verifyForgetOptions(opts *ForgetOptions) error {
 	return nil
 }
 
-func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, args []string) error {
+func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
 	err := verifyForgetOptions(&opts)
 	if err != nil {
 		return err
@@ -162,30 +166,31 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg
 		return err
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
-	if err != nil {
-		return err
-	}
-
 	if gopts.NoLock && !opts.DryRun {
 		return errors.Fatal("--no-lock is only applicable in combination with --dry-run for forget command")
 	}
 
-	if !opts.DryRun || !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun && gopts.NoLock)
+	if err != nil {
+		return err
 	}
+	defer unlock()
+
+	verbosity := gopts.verbosity
+	if gopts.JSON {
+		verbosity = 0
+	}
+	printer := newTerminalProgressPrinter(verbosity, term)
 
 	var snapshots restic.Snapshots
 	removeSnIDs := restic.NewIDSet()
 
-	for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) {
+	for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
 		snapshots = append(snapshots, sn)
 	}
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 
 	var jsonGroups []*ForgetGroup
 
@@ -217,15 +222,11 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg
 		}
 
 		if policy.Empty() && len(args) == 0 {
-			if !gopts.JSON {
-				Verbosef("no policy was specified, no snapshots will be removed\n")
-			}
+			printer.P("no policy was specified, no snapshots will be removed\n")
 		}
 
 		if !policy.Empty() {
-			if !gopts.JSON {
-				Verbosef("Applying Policy: %v\n", policy)
-			}
+			printer.P("Applying Policy: %v\n", policy)
 
 			for k, snapshotGroup := range snapshotGroups {
 				if gopts.Verbose >= 1 && !gopts.JSON {
@@ -248,20 +249,20 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg
 				keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy)
 
 				if len(keep) != 0 && !gopts.Quiet && !gopts.JSON {
-					Printf("keep %d snapshots:\n", len(keep))
+					printer.P("keep %d snapshots:\n", len(keep))
 					PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact)
-					Printf("\n")
+					printer.P("\n")
 				}
-				addJSONSnapshots(&fg.Keep, keep)
+				fg.Keep = asJSONSnapshots(keep)
 
 				if len(remove) != 0 && !gopts.Quiet && !gopts.JSON {
-					Printf("remove %d snapshots:\n", len(remove))
+					printer.P("remove %d snapshots:\n", len(remove))
 					PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact)
-					Printf("\n")
+					printer.P("\n")
 				}
-				addJSONSnapshots(&fg.Remove, remove)
+				fg.Remove = asJSONSnapshots(remove)
 
-				fg.Reasons = reasons
+				fg.Reasons = asJSONKeeps(reasons)
 
 				jsonGroups = append(jsonGroups, &fg)
 
@@ -272,16 +273,27 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg
 		}
 	}
 
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
+
 	if len(removeSnIDs) > 0 {
 		if !opts.DryRun {
-			err := DeleteFilesChecked(ctx, gopts, repo, removeSnIDs, restic.SnapshotFile)
+			bar := printer.NewCounter("files deleted")
+			err := restic.ParallelRemove(ctx, repo, removeSnIDs, restic.SnapshotFile, func(id restic.ID, err error) error {
+				if err != nil {
+					printer.E("unable to remove %v/%v from the repository\n", restic.SnapshotFile, id)
+				} else {
+					printer.VV("removed %v/%v\n", restic.SnapshotFile, id)
+				}
+				return nil
+			}, bar)
+			bar.Done()
 			if err != nil {
 				return err
 			}
 		} else {
-			if !gopts.JSON {
-				Printf("Would have removed the following snapshots:\n%v\n\n", removeSnIDs)
-			}
+			printer.P("Would have removed the following snapshots:\n%v\n\n", removeSnIDs)
 		}
 	}
 
@@ -293,15 +305,13 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg
 	}
 
 	if len(removeSnIDs) > 0 && opts.Prune {
-		if !gopts.JSON {
-			if opts.DryRun {
-				Verbosef("%d snapshots would be removed, running prune dry run\n", len(removeSnIDs))
-			} else {
-				Verbosef("%d snapshots have been removed, running prune\n", len(removeSnIDs))
-			}
+		if opts.DryRun {
+			printer.P("%d snapshots would be removed, running prune dry run\n", len(removeSnIDs))
+		} else {
+			printer.P("%d snapshots have been removed, running prune\n", len(removeSnIDs))
 		}
 		pruneOptions.DryRun = opts.DryRun
-		return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs)
+		return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs, term)
 	}
 
 	return nil
@@ -309,23 +319,47 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg
 
 // ForgetGroup helps to print what is forgotten in JSON.
 type ForgetGroup struct {
-	Tags    []string            `json:"tags"`
-	Host    string              `json:"host"`
-	Paths   []string            `json:"paths"`
-	Keep    []Snapshot          `json:"keep"`
-	Remove  []Snapshot          `json:"remove"`
-	Reasons []restic.KeepReason `json:"reasons"`
+	Tags    []string     `json:"tags"`
+	Host    string       `json:"host"`
+	Paths   []string     `json:"paths"`
+	Keep    []Snapshot   `json:"keep"`
+	Remove  []Snapshot   `json:"remove"`
+	Reasons []KeepReason `json:"reasons"`
 }
 
-func addJSONSnapshots(js *[]Snapshot, list restic.Snapshots) {
+func asJSONSnapshots(list restic.Snapshots) []Snapshot {
+	var resultList []Snapshot
 	for _, sn := range list {
 		k := Snapshot{
 			Snapshot: sn,
 			ID:       sn.ID(),
 			ShortID:  sn.ID().Str(),
 		}
-		*js = append(*js, k)
+		resultList = append(resultList, k)
 	}
+	return resultList
+}
+
+// KeepReason helps to print KeepReasons as JSON with Snapshots with their ID included.
+type KeepReason struct {
+	Snapshot Snapshot `json:"snapshot"`
+	Matches  []string `json:"matches"`
+}
+
+func asJSONKeeps(list []restic.KeepReason) []KeepReason {
+	var resultList []KeepReason
+	for _, keep := range list {
+		k := KeepReason{
+			Snapshot: Snapshot{
+				Snapshot: keep.Snapshot,
+				ID:       keep.Snapshot.ID(),
+				ShortID:  keep.Snapshot.ID().Str(),
+			},
+			Matches: keep.Matches,
+		}
+		resultList = append(resultList, k)
+	}
+	return resultList
 }
 
 func printJSONForget(stdout io.Writer, forgets []*ForgetGroup) error {

cmd/restic/cmd_forget_integration_test.go

@@ -5,9 +5,15 @@ import (
 	"testing"
 
 	rtest "github.com/restic/restic/internal/test"
+	"github.com/restic/restic/internal/ui/termstatus"
 )
 
 func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
 	opts := ForgetOptions{}
-	rtest.OK(t, runForget(context.TODO(), opts, gopts, args))
+	pruneOpts := PruneOptions{
+		MaxUnused: "5%",
+	}
+	rtest.OK(t, withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+		return runForget(context.TODO(), opts, pruneOpts, gopts, term, args)
+	}))
 }

cmd/restic/cmd_generate.go

@@ -21,7 +21,9 @@ EXIT STATUS
 Exit status is 0 if the command was successful, and non-zero if there was any error.
 `,
 	DisableAutoGenTag: true,
-	RunE:              runGenerate,
+	RunE: func(_ *cobra.Command, args []string) error {
+		return runGenerate(genOpts, args)
+	},
 }
 
 type generateOptions struct {
@@ -90,48 +92,48 @@ func writePowerShellCompletion(file string) error {
 	return cmdRoot.GenPowerShellCompletionFile(file)
 }
 
-func runGenerate(_ *cobra.Command, args []string) error {
+func runGenerate(opts generateOptions, args []string) error {
 	if len(args) > 0 {
 		return errors.Fatal("the generate command expects no arguments, only options - please see `restic help generate` for usage and flags")
 	}
 
-	if genOpts.ManDir != "" {
-		err := writeManpages(genOpts.ManDir)
+	if opts.ManDir != "" {
+		err := writeManpages(opts.ManDir)
 		if err != nil {
 			return err
 		}
 	}
 
-	if genOpts.BashCompletionFile != "" {
-		err := writeBashCompletion(genOpts.BashCompletionFile)
+	if opts.BashCompletionFile != "" {
+		err := writeBashCompletion(opts.BashCompletionFile)
 		if err != nil {
 			return err
 		}
 	}
 
-	if genOpts.FishCompletionFile != "" {
-		err := writeFishCompletion(genOpts.FishCompletionFile)
+	if opts.FishCompletionFile != "" {
+		err := writeFishCompletion(opts.FishCompletionFile)
 		if err != nil {
 			return err
 		}
 	}
 
-	if genOpts.ZSHCompletionFile != "" {
-		err := writeZSHCompletion(genOpts.ZSHCompletionFile)
+	if opts.ZSHCompletionFile != "" {
+		err := writeZSHCompletion(opts.ZSHCompletionFile)
 		if err != nil {
 			return err
 		}
 	}
 
-	if genOpts.PowerShellCompletionFile != "" {
-		err := writePowerShellCompletion(genOpts.PowerShellCompletionFile)
+	if opts.PowerShellCompletionFile != "" {
+		err := writePowerShellCompletion(opts.PowerShellCompletionFile)
 		if err != nil {
 			return err
 		}
 	}
 
 	var empty generateOptions
-	if genOpts == empty {
+	if opts == empty {
 		return errors.Fatal("nothing to do, please specify at least one output file/dir")
 	}
 

cmd/restic/cmd_init.go

@@ -80,7 +80,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
 		return err
 	}
 
-	gopts.password, err = ReadPasswordTwice(gopts,
+	gopts.password, err = ReadPasswordTwice(ctx, gopts,
 		"enter password for new repository: ",
 		"enter password again: ")
 	if err != nil {
@@ -131,7 +131,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
 
 func maybeReadChunkerPolynomial(ctx context.Context, opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) {
 	if opts.CopyChunkerParameters {
-		otherGopts, _, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary")
+		otherGopts, _, err := fillSecondaryGlobalOpts(ctx, opts.secondaryRepoOptions, gopts, "secondary")
 		if err != nil {
 			return nil, err
 		}

cmd/restic/cmd_key.go

@@ -1,265 +1,18 @@
 package main
 
 import (
-	"context"
-	"encoding/json"
-	"os"
-	"strings"
-	"sync"
-
-	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/repository"
-	"github.com/restic/restic/internal/restic"
-	"github.com/restic/restic/internal/ui/table"
-
 	"github.com/spf13/cobra"
 )
 
 var cmdKey = &cobra.Command{
-	Use:   "key [flags] [list|add|remove|passwd] [ID]",
+	Use:   "key",
 	Short: "Manage keys (passwords)",
 	Long: `
-The "key" command manages keys (passwords) for accessing the repository.
-
-EXIT STATUS
-===========
-
-Exit status is 0 if the command was successful, and non-zero if there was any error.
-`,
-	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return runKey(cmd.Context(), globalOptions, args)
-	},
+The "key" command allows you to set multiple access keys or passwords
+per repository.
+	`,
 }
 
-var (
-	newPasswordFile string
-	keyUsername     string
-	keyHostname     string
-)
-
 func init() {
 	cmdRoot.AddCommand(cmdKey)
-
-	flags := cmdKey.Flags()
-	flags.StringVarP(&newPasswordFile, "new-password-file", "", "", "`file` from which to read the new password")
-	flags.StringVarP(&keyUsername, "user", "", "", "the username for new keys")
-	flags.StringVarP(&keyHostname, "host", "", "", "the hostname for new keys")
-}
-
-func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions) error {
-	type keyInfo struct {
-		Current  bool   `json:"current"`
-		ID       string `json:"id"`
-		UserName string `json:"userName"`
-		HostName string `json:"hostName"`
-		Created  string `json:"created"`
-	}
-
-	var m sync.Mutex
-	var keys []keyInfo
-
-	err := restic.ParallelList(ctx, s.Backend(), restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
-		k, err := repository.LoadKey(ctx, s, id)
-		if err != nil {
-			Warnf("LoadKey() failed: %v\n", err)
-			return nil
-		}
-
-		key := keyInfo{
-			Current:  id == s.KeyID(),
-			ID:       id.Str(),
-			UserName: k.Username,
-			HostName: k.Hostname,
-			Created:  k.Created.Local().Format(TimeFormat),
-		}
-
-		m.Lock()
-		defer m.Unlock()
-		keys = append(keys, key)
-		return nil
-	})
-
-	if err != nil {
-		return err
-	}
-
-	if gopts.JSON {
-		return json.NewEncoder(globalOptions.stdout).Encode(keys)
-	}
-
-	tab := table.New()
-	tab.AddColumn(" ID", "{{if .Current}}*{{else}} {{end}}{{ .ID }}")
-	tab.AddColumn("User", "{{ .UserName }}")
-	tab.AddColumn("Host", "{{ .HostName }}")
-	tab.AddColumn("Created", "{{ .Created }}")
-
-	for _, key := range keys {
-		tab.AddRow(key)
-	}
-
-	return tab.Write(globalOptions.stdout)
-}
-
-// testKeyNewPassword is used to set a new password during integration testing.
-var testKeyNewPassword string
-
-func getNewPassword(gopts GlobalOptions) (string, error) {
-	if testKeyNewPassword != "" {
-		return testKeyNewPassword, nil
-	}
-
-	if newPasswordFile != "" {
-		return loadPasswordFromFile(newPasswordFile)
-	}
-
-	// Since we already have an open repository, temporary remove the password
-	// to prompt the user for the passwd.
-	newopts := gopts
-	newopts.password = ""
-
-	return ReadPasswordTwice(newopts,
-		"enter new password: ",
-		"enter password again: ")
-}
-
-func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error {
-	pw, err := getNewPassword(gopts)
-	if err != nil {
-		return err
-	}
-
-	id, err := repository.AddKey(ctx, repo, pw, keyUsername, keyHostname, repo.Key())
-	if err != nil {
-		return errors.Fatalf("creating new key failed: %v\n", err)
-	}
-
-	err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
-	if err != nil {
-		return err
-	}
-
-	Verbosef("saved new key as %s\n", id)
-
-	return nil
-}
-
-func deleteKey(ctx context.Context, repo *repository.Repository, id restic.ID) error {
-	if id == repo.KeyID() {
-		return errors.Fatal("refusing to remove key currently used to access repository")
-	}
-
-	h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
-	err := repo.Backend().Remove(ctx, h)
-	if err != nil {
-		return err
-	}
-
-	Verbosef("removed key %v\n", id)
-	return nil
-}
-
-func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error {
-	pw, err := getNewPassword(gopts)
-	if err != nil {
-		return err
-	}
-
-	id, err := repository.AddKey(ctx, repo, pw, "", "", repo.Key())
-	if err != nil {
-		return errors.Fatalf("creating new key failed: %v\n", err)
-	}
-	oldID := repo.KeyID()
-
-	err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
-	if err != nil {
-		return err
-	}
-
-	h := restic.Handle{Type: restic.KeyFile, Name: oldID.String()}
-	err = repo.Backend().Remove(ctx, h)
-	if err != nil {
-		return err
-	}
-
-	Verbosef("saved new key as %s\n", id)
-
-	return nil
-}
-
-func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error {
-	// Verify new key to make sure it really works. A broken key can render the
-	// whole repository inaccessible
-	err := repo.SearchKey(ctx, pw, 0, key.ID().String())
-	if err != nil {
-		// the key is invalid, try to remove it
-		h := restic.Handle{Type: restic.KeyFile, Name: key.ID().String()}
-		_ = repo.Backend().Remove(ctx, h)
-		return errors.Fatalf("failed to access repository with new key: %v", err)
-	}
-	return nil
-}
-
-func runKey(ctx context.Context, gopts GlobalOptions, args []string) error {
-	if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) {
-		return errors.Fatal("wrong number of arguments")
-	}
-
-	repo, err := OpenRepository(ctx, gopts)
-	if err != nil {
-		return err
-	}
-
-	switch args[0] {
-	case "list":
-		if !gopts.NoLock {
-			var lock *restic.Lock
-			lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-			defer unlockRepo(lock)
-			if err != nil {
-				return err
-			}
-		}
-
-		return listKeys(ctx, repo, gopts)
-	case "add":
-		lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-
-		return addKey(ctx, repo, gopts)
-	case "remove":
-		lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-
-		id, err := restic.Find(ctx, repo.Backend(), restic.KeyFile, args[1])
-		if err != nil {
-			return err
-		}
-
-		return deleteKey(ctx, repo, id)
-	case "passwd":
-		lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-
-		return changePassword(ctx, repo, gopts)
-	}
-
-	return nil
-}
-
-func loadPasswordFromFile(pwdFile string) (string, error) {
-	s, err := os.ReadFile(pwdFile)
-	if os.IsNotExist(err) {
-		return "", errors.Fatalf("%s does not exist", pwdFile)
-	}
-	return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile")
 }

cmd/restic/cmd_key_add.go

@@ -0,0 +1,123 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository"
+	"github.com/spf13/cobra"
+)
+
+var cmdKeyAdd = &cobra.Command{
+	Use:   "add",
+	Short: "Add a new key (password) to the repository; returns the new key ID",
+	Long: `
+The "add" sub-command creates a new key and validates the key. Returns the new key ID.
+
+EXIT STATUS
+===========
+
+Exit status is 0 if the command is successful, and non-zero if there was any error.
+	`,
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runKeyAdd(cmd.Context(), globalOptions, keyAddOpts, args)
+	},
+}
+
+type KeyAddOptions struct {
+	NewPasswordFile string
+	Username        string
+	Hostname        string
+}
+
+var keyAddOpts KeyAddOptions
+
+func init() {
+	cmdKey.AddCommand(cmdKeyAdd)
+
+	flags := cmdKeyAdd.Flags()
+	flags.StringVarP(&keyAddOpts.NewPasswordFile, "new-password-file", "", "", "`file` from which to read the new password")
+	flags.StringVarP(&keyAddOpts.Username, "user", "", "", "the username for new key")
+	flags.StringVarP(&keyAddOpts.Hostname, "host", "", "", "the hostname for new key")
+}
+
+func runKeyAdd(ctx context.Context, gopts GlobalOptions, opts KeyAddOptions, args []string) error {
+	if len(args) > 0 {
+		return fmt.Errorf("the key add command expects no arguments, only options - please see `restic help key add` for usage and flags")
+	}
+
+	ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, false)
+	if err != nil {
+		return err
+	}
+	defer unlock()
+
+	return addKey(ctx, repo, gopts, opts)
+}
+
+func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyAddOptions) error {
+	pw, err := getNewPassword(ctx, gopts, opts.NewPasswordFile)
+	if err != nil {
+		return err
+	}
+
+	id, err := repository.AddKey(ctx, repo, pw, opts.Username, opts.Hostname, repo.Key())
+	if err != nil {
+		return errors.Fatalf("creating new key failed: %v\n", err)
+	}
+
+	err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
+	if err != nil {
+		return err
+	}
+
+	Verbosef("saved new key with ID %s\n", id.ID())
+
+	return nil
+}
+
+// testKeyNewPassword is used to set a new password during integration testing.
+var testKeyNewPassword string
+
+func getNewPassword(ctx context.Context, gopts GlobalOptions, newPasswordFile string) (string, error) {
+	if testKeyNewPassword != "" {
+		return testKeyNewPassword, nil
+	}
+
+	if newPasswordFile != "" {
+		return loadPasswordFromFile(newPasswordFile)
+	}
+
+	// Since we already have an open repository, temporary remove the password
+	// to prompt the user for the passwd.
+	newopts := gopts
+	newopts.password = ""
+
+	return ReadPasswordTwice(ctx, newopts,
+		"enter new password: ",
+		"enter password again: ")
+}
+
+func loadPasswordFromFile(pwdFile string) (string, error) {
+	s, err := os.ReadFile(pwdFile)
+	if os.IsNotExist(err) {
+		return "", errors.Fatalf("%s does not exist", pwdFile)
+	}
+	return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile")
+}
+
+func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error {
+	// Verify new key to make sure it really works. A broken key can render the
+	// whole repository inaccessible
+	err := repo.SearchKey(ctx, pw, 0, key.ID().String())
+	if err != nil {
+		// the key is invalid, try to remove it
+		_ = repository.RemoveKey(ctx, repo, key.ID())
+		return errors.Fatalf("failed to access repository with new key: %v", err)
+	}
+	return nil
+}

cmd/restic/cmd_key_integration_test.go

@@ -4,16 +4,17 @@ import (
 	"bufio"
 	"context"
 	"regexp"
+	"strings"
 	"testing"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/repository"
-	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )
 
 func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string {
 	buf, err := withCaptureStdout(func() error {
-		return runKey(context.TODO(), gopts, []string{"list"})
+		return runKeyList(context.TODO(), gopts, []string{})
 	})
 	rtest.OK(t, err)
 
@@ -36,21 +37,20 @@ func testRunKeyAddNewKey(t testing.TB, newPassword string, gopts GlobalOptions)
 		testKeyNewPassword = ""
 	}()
 
-	rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"}))
+	rtest.OK(t, runKeyAdd(context.TODO(), gopts, KeyAddOptions{}, []string{}))
 }
 
 func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) {
 	testKeyNewPassword = "john's geheimnis"
 	defer func() {
 		testKeyNewPassword = ""
-		keyUsername = ""
-		keyHostname = ""
 	}()
 
-	rtest.OK(t, cmdKey.Flags().Parse([]string{"--user=john", "--host=example.com"}))
-
 	t.Log("adding key for john@example.com")
-	rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"}))
+	rtest.OK(t, runKeyAdd(context.TODO(), gopts, KeyAddOptions{
+		Username: "john",
+		Hostname: "example.com",
+	}, []string{}))
 
 	repo, err := OpenRepository(context.TODO(), gopts)
 	rtest.OK(t, err)
@@ -67,13 +67,13 @@ func testRunKeyPasswd(t testing.TB, newPassword string, gopts GlobalOptions) {
 		testKeyNewPassword = ""
 	}()
 
-	rtest.OK(t, runKey(context.TODO(), gopts, []string{"passwd"}))
+	rtest.OK(t, runKeyPasswd(context.TODO(), gopts, KeyPasswdOptions{}, []string{}))
 }
 
 func testRunKeyRemove(t testing.TB, gopts GlobalOptions, IDs []string) {
 	t.Logf("remove %d keys: %q\n", len(IDs), IDs)
 	for _, id := range IDs {
-		rtest.OK(t, runKey(context.TODO(), gopts, []string{"remove", id}))
+		rtest.OK(t, runKeyRemove(context.TODO(), gopts, []string{id}))
 	}
 }
 
@@ -103,18 +103,18 @@ func TestKeyAddRemove(t *testing.T) {
 
 	env.gopts.password = passwordList[len(passwordList)-1]
 	t.Logf("testing access with last password %q\n", env.gopts.password)
-	rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"}))
+	rtest.OK(t, runKeyList(context.TODO(), env.gopts, []string{}))
 	testRunCheck(t, env.gopts)
 
 	testRunKeyAddNewKeyUserHost(t, env.gopts)
 }
 
 type emptySaveBackend struct {
-	restic.Backend
+	backend.Backend
 }
 
-func (b *emptySaveBackend) Save(ctx context.Context, h restic.Handle, _ restic.RewindReader) error {
-	return b.Backend.Save(ctx, h, restic.NewByteReader([]byte{}, nil))
+func (b *emptySaveBackend) Save(ctx context.Context, h backend.Handle, _ backend.RewindReader) error {
+	return b.Backend.Save(ctx, h, backend.NewByteReader([]byte{}, nil))
 }
 
 func TestKeyProblems(t *testing.T) {
@@ -122,7 +122,7 @@ func TestKeyProblems(t *testing.T) {
 	defer cleanup()
 
 	testRunInit(t, env.gopts)
-	env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
+	env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
 		return &emptySaveBackend{r}, nil
 	}
 
@@ -131,15 +131,45 @@ func TestKeyProblems(t *testing.T) {
 		testKeyNewPassword = ""
 	}()
 
-	err := runKey(context.TODO(), env.gopts, []string{"passwd"})
+	err := runKeyPasswd(context.TODO(), env.gopts, KeyPasswdOptions{}, []string{})
 	t.Log(err)
 	rtest.Assert(t, err != nil, "expected passwd change to fail")
 
-	err = runKey(context.TODO(), env.gopts, []string{"add"})
+	err = runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{}, []string{})
 	t.Log(err)
 	rtest.Assert(t, err != nil, "expected key adding to fail")
 
 	t.Logf("testing access with initial password %q\n", env.gopts.password)
-	rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"}))
+	rtest.OK(t, runKeyList(context.TODO(), env.gopts, []string{}))
 	testRunCheck(t, env.gopts)
 }
+
+func TestKeyCommandInvalidArguments(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+	env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
+		return &emptySaveBackend{r}, nil
+	}
+
+	err := runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{}, []string{"johndoe"})
+	t.Log(err)
+	rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key add: %v", err)
+
+	err = runKeyPasswd(context.TODO(), env.gopts, KeyPasswdOptions{}, []string{"johndoe"})
+	t.Log(err)
+	rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key passwd: %v", err)
+
+	err = runKeyList(context.TODO(), env.gopts, []string{"johndoe"})
+	t.Log(err)
+	rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key list: %v", err)
+
+	err = runKeyRemove(context.TODO(), env.gopts, []string{})
+	t.Log(err)
+	rtest.Assert(t, err != nil && strings.Contains(err.Error(), "one argument"), "unexpected error for key remove: %v", err)
+
+	err = runKeyRemove(context.TODO(), env.gopts, []string{"john", "doe"})
+	t.Log(err)
+	rtest.Assert(t, err != nil && strings.Contains(err.Error(), "one argument"), "unexpected error for key remove: %v", err)
+}

cmd/restic/cmd_key_list.go

@@ -0,0 +1,106 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui/table"
+	"github.com/spf13/cobra"
+)
+
+var cmdKeyList = &cobra.Command{
+	Use:   "list",
+	Short: "List keys (passwords)",
+	Long: `
+The "list" sub-command lists all the keys (passwords) associated with the repository.
+Returns the key ID, username, hostname, created time and if it's the current key being
+used to access the repository.
+
+EXIT STATUS
+===========
+
+Exit status is 0 if the command is successful, and non-zero if there was any error.
+	`,
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runKeyList(cmd.Context(), globalOptions, args)
+	},
+}
+
+func init() {
+	cmdKey.AddCommand(cmdKeyList)
+}
+
+func runKeyList(ctx context.Context, gopts GlobalOptions, args []string) error {
+	if len(args) > 0 {
+		return fmt.Errorf("the key list command expects no arguments, only options - please see `restic help key list` for usage and flags")
+	}
+
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
+	if err != nil {
+		return err
+	}
+	defer unlock()
+
+	return listKeys(ctx, repo, gopts)
+}
+
+func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions) error {
+	type keyInfo struct {
+		Current  bool   `json:"current"`
+		ID       string `json:"id"`
+		ShortID  string `json:"-"`
+		UserName string `json:"userName"`
+		HostName string `json:"hostName"`
+		Created  string `json:"created"`
+	}
+
+	var m sync.Mutex
+	var keys []keyInfo
+
+	err := restic.ParallelList(ctx, s, restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, _ int64) error {
+		k, err := repository.LoadKey(ctx, s, id)
+		if err != nil {
+			Warnf("LoadKey() failed: %v\n", err)
+			return nil
+		}
+
+		key := keyInfo{
+			Current:  id == s.KeyID(),
+			ID:       id.String(),
+			ShortID:  id.Str(),
+			UserName: k.Username,
+			HostName: k.Hostname,
+			Created:  k.Created.Local().Format(TimeFormat),
+		}
+
+		m.Lock()
+		defer m.Unlock()
+		keys = append(keys, key)
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	if gopts.JSON {
+		return json.NewEncoder(globalOptions.stdout).Encode(keys)
+	}
+
+	tab := table.New()
+	tab.AddColumn(" ID", "{{if .Current}}*{{else}} {{end}}{{ .ShortID }}")
+	tab.AddColumn("User", "{{ .UserName }}")
+	tab.AddColumn("Host", "{{ .HostName }}")
+	tab.AddColumn("Created", "{{ .Created }}")
+
+	for _, key := range keys {
+		tab.AddRow(key)
+	}
+
+	return tab.Write(globalOptions.stdout)
+}

cmd/restic/cmd_key_passwd.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository"
+	"github.com/spf13/cobra"
+)
+
+var cmdKeyPasswd = &cobra.Command{
+	Use:   "passwd",
+	Short: "Change key (password); creates a new key ID and removes the old key ID, returns new key ID",
+	Long: `
+The "passwd" sub-command creates a new key, validates the key and remove the old key ID.
+Returns the new key ID. 
+
+EXIT STATUS
+===========
+
+Exit status is 0 if the command is successful, and non-zero if there was any error.
+	`,
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runKeyPasswd(cmd.Context(), globalOptions, keyPasswdOpts, args)
+	},
+}
+
+type KeyPasswdOptions struct {
+	KeyAddOptions
+}
+
+var keyPasswdOpts KeyPasswdOptions
+
+func init() {
+	cmdKey.AddCommand(cmdKeyPasswd)
+
+	flags := cmdKeyPasswd.Flags()
+	flags.StringVarP(&keyPasswdOpts.NewPasswordFile, "new-password-file", "", "", "`file` from which to read the new password")
+	flags.StringVarP(&keyPasswdOpts.Username, "user", "", "", "the username for new key")
+	flags.StringVarP(&keyPasswdOpts.Hostname, "host", "", "", "the hostname for new key")
+}
+
+func runKeyPasswd(ctx context.Context, gopts GlobalOptions, opts KeyPasswdOptions, args []string) error {
+	if len(args) > 0 {
+		return fmt.Errorf("the key passwd command expects no arguments, only options - please see `restic help key passwd` for usage and flags")
+	}
+
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
+	if err != nil {
+		return err
+	}
+	defer unlock()
+
+	return changePassword(ctx, repo, gopts, opts)
+}
+
+func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyPasswdOptions) error {
+	pw, err := getNewPassword(ctx, gopts, opts.NewPasswordFile)
+	if err != nil {
+		return err
+	}
+
+	id, err := repository.AddKey(ctx, repo, pw, "", "", repo.Key())
+	if err != nil {
+		return errors.Fatalf("creating new key failed: %v\n", err)
+	}
+	oldID := repo.KeyID()
+
+	err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
+	if err != nil {
+		return err
+	}
+
+	err = repository.RemoveKey(ctx, repo, oldID)
+	if err != nil {
+		return err
+	}
+
+	Verbosef("saved new key as %s\n", id)
+
+	return nil
+}

cmd/restic/cmd_key_remove.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/restic/restic/internal/errors"
+	"github.com/restic/restic/internal/repository"
+	"github.com/restic/restic/internal/restic"
+	"github.com/spf13/cobra"
+)
+
+var cmdKeyRemove = &cobra.Command{
+	Use:   "remove [ID]",
+	Short: "Remove key ID (password) from the repository.",
+	Long: `
+The "remove" sub-command removes the selected key ID. The "remove" command does not allow
+removing the current key being used to access the repository. 
+
+EXIT STATUS
+===========
+
+Exit status is 0 if the command is successful, and non-zero if there was any error.
+	`,
+	DisableAutoGenTag: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runKeyRemove(cmd.Context(), globalOptions, args)
+	},
+}
+
+func init() {
+	cmdKey.AddCommand(cmdKeyRemove)
+}
+
+func runKeyRemove(ctx context.Context, gopts GlobalOptions, args []string) error {
+	if len(args) != 1 {
+		return fmt.Errorf("key remove expects one argument as the key id")
+	}
+
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
+	if err != nil {
+		return err
+	}
+	defer unlock()
+
+	return deleteKey(ctx, repo, args[0])
+}
+
+func deleteKey(ctx context.Context, repo *repository.Repository, idPrefix string) error {
+	id, err := restic.Find(ctx, repo, restic.KeyFile, idPrefix)
+	if err != nil {
+		return err
+	}
+
+	if id == repo.KeyID() {
+		return errors.Fatal("refusing to remove key currently used to access repository")
+	}
+
+	err = repository.RemoveKey(ctx, repo, id)
+	if err != nil {
+		return err
+	}
+
+	Verbosef("removed key %v\n", id)
+	return nil
+}

cmd/restic/cmd_list.go

@@ -23,7 +23,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runList(cmd.Context(), cmd, globalOptions, args)
+		return runList(cmd.Context(), globalOptions, args)
 	},
 }
 
@@ -31,24 +31,16 @@ func init() {
 	cmdRoot.AddCommand(cmdList)
 }
 
-func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args []string) error {
+func runList(ctx context.Context, gopts GlobalOptions, args []string) error {
 	if len(args) != 1 {
-		return errors.Fatal("type not specified, usage: " + cmd.Use)
+		return errors.Fatal("type not specified")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock || args[0] == "locks")
 	if err != nil {
 		return err
 	}
-
-	if !gopts.NoLock && args[0] != "locks" {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
+	defer unlock()
 
 	var t restic.FileType
 	switch args[0] {
@@ -63,20 +55,19 @@ func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args
 	case "locks":
 		t = restic.LockFile
 	case "blobs":
-		return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
+		return index.ForAllIndexes(ctx, repo, repo, func(_ restic.ID, idx *index.Index, _ bool, err error) error {
 			if err != nil {
 				return err
 			}
-			idx.Each(ctx, func(blobs restic.PackedBlob) {
+			return idx.Each(ctx, func(blobs restic.PackedBlob) {
 				Printf("%v %v\n", blobs.Type, blobs.ID)
 			})
-			return nil
 		})
 	default:
 		return errors.Fatal("invalid type")
 	}
 
-	return repo.List(ctx, t, func(id restic.ID, size int64) error {
+	return repo.List(ctx, t, func(id restic.ID, _ int64) error {
 		Printf("%s\n", id)
 		return nil
 	})

cmd/restic/cmd_list_integration_test.go

@@ -12,7 +12,7 @@ import (
 
 func testRunList(t testing.TB, tpe string, opts GlobalOptions) restic.IDs {
 	buf, err := withCaptureStdout(func() error {
-		return runList(context.TODO(), cmdList, opts, []string{tpe})
+		return runList(context.TODO(), opts, []string{tpe})
 	})
 	rtest.OK(t, err)
 	return parseIDsFromReader(t, buf)

cmd/restic/cmd_ls.go

@@ -3,13 +3,14 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"fmt"
+	"io"
 	"os"
 	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
 
-	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/fs"
 	"github.com/restic/restic/internal/restic"
@@ -52,6 +53,7 @@ type LsOptions struct {
 	restic.SnapshotFilter
 	Recursive     bool
 	HumanReadable bool
+	Ncdu          bool
 }
 
 var lsOptions LsOptions
@@ -64,16 +66,49 @@ func init() {
 	flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
 	flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
 	flags.BoolVar(&lsOptions.HumanReadable, "human-readable", false, "print sizes in human readable format")
+	flags.BoolVar(&lsOptions.Ncdu, "ncdu", false, "output NCDU export format (pipe into 'ncdu -f -')")
 }
 
-type lsSnapshot struct {
-	*restic.Snapshot
-	ID         *restic.ID `json:"id"`
-	ShortID    string     `json:"short_id"`
-	StructType string     `json:"struct_type"` // "snapshot"
+type lsPrinter interface {
+	Snapshot(sn *restic.Snapshot)
+	Node(path string, node *restic.Node)
+	LeaveDir(path string)
+	Close()
+}
+
+type jsonLsPrinter struct {
+	enc *json.Encoder
+}
+
+func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) {
+	type lsSnapshot struct {
+		*restic.Snapshot
+		ID          *restic.ID `json:"id"`
+		ShortID     string     `json:"short_id"`
+		MessageType string     `json:"message_type"` // "snapshot"
+		StructType  string     `json:"struct_type"`  // "snapshot", deprecated
+	}
+
+	err := p.enc.Encode(lsSnapshot{
+		Snapshot:    sn,
+		ID:          sn.ID(),
+		ShortID:     sn.ID().Str(),
+		MessageType: "snapshot",
+		StructType:  "snapshot",
+	})
+	if err != nil {
+		Warnf("JSON encode failed: %v\n", err)
+	}
 }
 
 // Print node in our custom JSON format, followed by a newline.
+func (p *jsonLsPrinter) Node(path string, node *restic.Node) {
+	err := lsNodeJSON(p.enc, path, node)
+	if err != nil {
+		Warnf("JSON encode failed: %v\n", err)
+	}
+}
+
 func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 	n := &struct {
 		Name        string      `json:"name"`
@@ -88,7 +123,8 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 		AccessTime  time.Time   `json:"atime,omitempty"`
 		ChangeTime  time.Time   `json:"ctime,omitempty"`
 		Inode       uint64      `json:"inode,omitempty"`
-		StructType  string      `json:"struct_type"` // "node"
+		MessageType string      `json:"message_type"` // "node"
+		StructType  string      `json:"struct_type"`  // "node", deprecated
 
 		size uint64 // Target for Size pointer.
 	}{
@@ -104,6 +140,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 		AccessTime:  node.AccessTime,
 		ChangeTime:  node.ChangeTime,
 		Inode:       node.Inode,
+		MessageType: "node",
 		StructType:  "node",
 	}
 	// Always print size for regular files, even when empty,
@@ -115,10 +152,117 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
 	return enc.Encode(n)
 }
 
+func (p *jsonLsPrinter) LeaveDir(_ string) {}
+func (p *jsonLsPrinter) Close()            {}
+
+type ncduLsPrinter struct {
+	out   io.Writer
+	depth int
+}
+
+// lsSnapshotNcdu prints a restic snapshot in Ncdu save format.
+// It opens the JSON list. Nodes are added with lsNodeNcdu and the list is closed by lsCloseNcdu.
+// Format documentation: https://dev.yorhel.nl/ncdu/jsonfmt
+func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) {
+	const NcduMajorVer = 1
+	const NcduMinorVer = 2
+
+	snapshotBytes, err := json.Marshal(sn)
+	if err != nil {
+		Warnf("JSON encode failed: %v\n", err)
+	}
+	p.depth++
+	fmt.Fprintf(p.out, "[%d, %d, %s", NcduMajorVer, NcduMinorVer, string(snapshotBytes))
+}
+
+func lsNcduNode(_ string, node *restic.Node) ([]byte, error) {
+	type NcduNode struct {
+		Name   string `json:"name"`
+		Asize  uint64 `json:"asize"`
+		Dsize  uint64 `json:"dsize"`
+		Dev    uint64 `json:"dev"`
+		Ino    uint64 `json:"ino"`
+		NLink  uint64 `json:"nlink"`
+		NotReg bool   `json:"notreg"`
+		UID    uint32 `json:"uid"`
+		GID    uint32 `json:"gid"`
+		Mode   uint16 `json:"mode"`
+		Mtime  int64  `json:"mtime"`
+	}
+
+	outNode := NcduNode{
+		Name:   node.Name,
+		Asize:  node.Size,
+		Dsize:  node.Size,
+		Dev:    node.DeviceID,
+		Ino:    node.Inode,
+		NLink:  node.Links,
+		NotReg: node.Type != "dir" && node.Type != "file",
+		UID:    node.UID,
+		GID:    node.GID,
+		Mode:   uint16(node.Mode & os.ModePerm),
+		Mtime:  node.ModTime.Unix(),
+	}
+	//  bits according to inode(7) manpage
+	if node.Mode&os.ModeSetuid != 0 {
+		outNode.Mode |= 0o4000
+	}
+	if node.Mode&os.ModeSetgid != 0 {
+		outNode.Mode |= 0o2000
+	}
+	if node.Mode&os.ModeSticky != 0 {
+		outNode.Mode |= 0o1000
+	}
+
+	return json.Marshal(outNode)
+}
+
+func (p *ncduLsPrinter) Node(path string, node *restic.Node) {
+	out, err := lsNcduNode(path, node)
+	if err != nil {
+		Warnf("JSON encode failed: %v\n", err)
+	}
+
+	if node.Type == "dir" {
+		fmt.Fprintf(p.out, ",\n%s[\n%s%s", strings.Repeat("  ", p.depth), strings.Repeat("  ", p.depth+1), string(out))
+		p.depth++
+	} else {
+		fmt.Fprintf(p.out, ",\n%s%s", strings.Repeat("  ", p.depth), string(out))
+	}
+}
+
+func (p *ncduLsPrinter) LeaveDir(_ string) {
+	p.depth--
+	fmt.Fprintf(p.out, "\n%s]", strings.Repeat("  ", p.depth))
+}
+
+func (p *ncduLsPrinter) Close() {
+	fmt.Fprint(p.out, "\n]\n")
+}
+
+type textLsPrinter struct {
+	dirs          []string
+	ListLong      bool
+	HumanReadable bool
+}
+
+func (p *textLsPrinter) Snapshot(sn *restic.Snapshot) {
+	Verbosef("%v filtered by %v:\n", sn, p.dirs)
+}
+func (p *textLsPrinter) Node(path string, node *restic.Node) {
+	Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable))
+}
+
+func (p *textLsPrinter) LeaveDir(_ string) {}
+func (p *textLsPrinter) Close()            {}
+
 func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error {
 	if len(args) == 0 {
 		return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'")
 	}
+	if opts.Ncdu && gopts.JSON {
+		return errors.Fatal("only either '--json' or '--ncdu' can be specified")
+	}
 
 	// extract any specific directories to walk
 	var dirs []string
@@ -165,12 +309,13 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		return false
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
+	defer unlock()
 
-	snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
+	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
@@ -180,38 +325,21 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		return err
 	}
 
-	var (
-		printSnapshot func(sn *restic.Snapshot)
-		printNode     func(path string, node *restic.Node)
-	)
+	var printer lsPrinter
 
 	if gopts.JSON {
-		enc := json.NewEncoder(globalOptions.stdout)
-
-		printSnapshot = func(sn *restic.Snapshot) {
-			err := enc.Encode(lsSnapshot{
-				Snapshot:   sn,
-				ID:         sn.ID(),
-				ShortID:    sn.ID().Str(),
-				StructType: "snapshot",
-			})
-			if err != nil {
-				Warnf("JSON encode failed: %v\n", err)
-			}
+		printer = &jsonLsPrinter{
+			enc: json.NewEncoder(globalOptions.stdout),
 		}
-
-		printNode = func(path string, node *restic.Node) {
-			err := lsNodeJSON(enc, path, node)
-			if err != nil {
-				Warnf("JSON encode failed: %v\n", err)
-			}
+	} else if opts.Ncdu {
+		printer = &ncduLsPrinter{
+			out: globalOptions.stdout,
 		}
 	} else {
-		printSnapshot = func(sn *restic.Snapshot) {
-			Verbosef("snapshot %s of %v filtered by %v at %s):\n", sn.ID().Str(), sn.Paths, dirs, sn.Time)
-		}
-		printNode = func(path string, node *restic.Node) {
-			Printf("%s\n", formatNode(path, node, lsOptions.ListLong, lsOptions.HumanReadable))
+		printer = &textLsPrinter{
+			dirs:          dirs,
+			ListLong:      opts.ListLong,
+			HumanReadable: opts.HumanReadable,
 		}
 	}
 
@@ -229,44 +357,55 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
 		return err
 	}
 
-	printSnapshot(sn)
+	printer.Snapshot(sn)
 
-	err = walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
+	processNode := func(_ restic.ID, nodepath string, node *restic.Node, err error) error {
 		if err != nil {
-			return false, err
+			return err
 		}
 		if node == nil {
-			return false, nil
+			return nil
 		}
 
 		if withinDir(nodepath) {
 			// if we're within a dir, print the node
-			printNode(nodepath, node)
+			printer.Node(nodepath, node)
 
 			// if recursive listing is requested, signal the walker that it
 			// should continue walking recursively
 			if opts.Recursive {
-				return false, nil
+				return nil
 			}
 		}
 
 		// if there's an upcoming match deeper in the tree (but we're not
 		// there yet), signal the walker to descend into any subdirs
 		if approachingMatchingTree(nodepath) {
-			return false, nil
+			return nil
 		}
 
 		// otherwise, signal the walker to not walk recursively into any
 		// subdirs
 		if node.Type == "dir" {
-			return false, walker.ErrSkipNode
+			return walker.ErrSkipNode
 		}
-		return false, nil
+		return nil
+	}
+
+	err = walker.Walk(ctx, repo, *sn.Tree, walker.WalkVisitor{
+		ProcessNode: processNode,
+		LeaveDir: func(path string) {
+			// the root path `/` has no corresponding node and is thus also skipped by processNode
+			if withinDir(path) && path != "/" {
+				printer.LeaveDir(path)
+			}
+		},
 	})
 
 	if err != nil {
 		return err
 	}
 
+	printer.Close()
 	return nil
 }

cmd/restic/cmd_ls_integration_test.go

@@ -2,18 +2,46 @@ package main
 
 import (
 	"context"
+	"encoding/json"
+	"path/filepath"
 	"strings"
 	"testing"
 
 	rtest "github.com/restic/restic/internal/test"
 )
 
-func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
+func testRunLsWithOpts(t testing.TB, gopts GlobalOptions, opts LsOptions, args []string) []byte {
 	buf, err := withCaptureStdout(func() error {
 		gopts.Quiet = true
-		opts := LsOptions{}
-		return runLs(context.TODO(), opts, gopts, []string{snapshotID})
+		return runLs(context.TODO(), opts, gopts, args)
 	})
 	rtest.OK(t, err)
-	return strings.Split(buf.String(), "\n")
+	return buf.Bytes()
+}
+
+func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
+	out := testRunLsWithOpts(t, gopts, LsOptions{}, []string{snapshotID})
+	return strings.Split(string(out), "\n")
+}
+
+func assertIsValidJSON(t *testing.T, data []byte) {
+	// Sanity check: output must be valid JSON.
+	var v interface{}
+	err := json.Unmarshal(data, &v)
+	rtest.OK(t, err)
+}
+
+func TestRunLsNcdu(t *testing.T) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+
+	testRunInit(t, env.gopts)
+	opts := BackupOptions{}
+	testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
+
+	ncdu := testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest"})
+	assertIsValidJSON(t, ncdu)
+
+	ncdu = testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest", "/testdata"})
+	assertIsValidJSON(t, ncdu)
 }

cmd/restic/cmd_ls_test.go

@@ -11,78 +11,94 @@ import (
 	rtest "github.com/restic/restic/internal/test"
 )
 
+type lsTestNode struct {
+	path string
+	restic.Node
+}
+
+var lsTestNodes = []lsTestNode{
+	// Mode is omitted when zero.
+	// Permissions, by convention is "-" per mode bit
+	{
+		path: "/bar/baz",
+		Node: restic.Node{
+			Name: "baz",
+			Type: "file",
+			Size: 12345,
+			UID:  10000000,
+			GID:  20000000,
+
+			User:  "nobody",
+			Group: "nobodies",
+			Links: 1,
+		},
+	},
+
+	// Even empty files get an explicit size.
+	{
+		path: "/foo/empty",
+		Node: restic.Node{
+			Name: "empty",
+			Type: "file",
+			Size: 0,
+			UID:  1001,
+			GID:  1001,
+
+			User:  "not printed",
+			Group: "not printed",
+			Links: 0xF00,
+		},
+	},
+
+	// Non-regular files do not get a size.
+	// Mode is printed in decimal, including the type bits.
+	{
+		path: "/foo/link",
+		Node: restic.Node{
+			Name:       "link",
+			Type:       "symlink",
+			Mode:       os.ModeSymlink | 0777,
+			LinkTarget: "not printed",
+		},
+	},
+
+	{
+		path: "/some/directory",
+		Node: restic.Node{
+			Name:       "directory",
+			Type:       "dir",
+			Mode:       os.ModeDir | 0755,
+			ModTime:    time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC),
+			AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC),
+			ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC),
+		},
+	},
+
+	// Test encoding of setuid/setgid/sticky bit
+	{
+		path: "/some/sticky",
+		Node: restic.Node{
+			Name: "sticky",
+			Type: "dir",
+			Mode: os.ModeDir | 0755 | os.ModeSetuid | os.ModeSetgid | os.ModeSticky,
+		},
+	},
+}
+
 func TestLsNodeJSON(t *testing.T) {
-	for _, c := range []struct {
-		path string
-		restic.Node
-		expect string
-	}{
-		// Mode is omitted when zero.
-		// Permissions, by convention is "-" per mode bit
-		{
-			path: "/bar/baz",
-			Node: restic.Node{
-				Name: "baz",
-				Type: "file",
-				Size: 12345,
-				UID:  10000000,
-				GID:  20000000,
-
-				User:  "nobody",
-				Group: "nobodies",
-				Links: 1,
-			},
-			expect: `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
-		},
-
-		// Even empty files get an explicit size.
-		{
-			path: "/foo/empty",
-			Node: restic.Node{
-				Name: "empty",
-				Type: "file",
-				Size: 0,
-				UID:  1001,
-				GID:  1001,
-
-				User:  "not printed",
-				Group: "not printed",
-				Links: 0xF00,
-			},
-			expect: `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
-		},
-
-		// Non-regular files do not get a size.
-		// Mode is printed in decimal, including the type bits.
-		{
-			path: "/foo/link",
-			Node: restic.Node{
-				Name:       "link",
-				Type:       "symlink",
-				Mode:       os.ModeSymlink | 0777,
-				LinkTarget: "not printed",
-			},
-			expect: `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
-		},
-
-		{
-			path: "/some/directory",
-			Node: restic.Node{
-				Name:       "directory",
-				Type:       "dir",
-				Mode:       os.ModeDir | 0755,
-				ModTime:    time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC),
-				AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC),
-				ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC),
-			},
-			expect: `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`,
-		},
+	for i, expect := range []string{
+		`{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","message_type":"node","struct_type":"node"}`,
+		`{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","message_type":"node","struct_type":"node"}`,
+		`{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","message_type":"node","struct_type":"node"}`,
+		`{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","message_type":"node","struct_type":"node"}`,
+		`{"name":"sticky","type":"dir","path":"/some/sticky","uid":0,"gid":0,"mode":2161115629,"permissions":"dugtrwxr-xr-x","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","message_type":"node","struct_type":"node"}`,
 	} {
+		c := lsTestNodes[i]
 		buf := new(bytes.Buffer)
 		enc := json.NewEncoder(buf)
 		err := lsNodeJSON(enc, c.path, &c.Node)
 		rtest.OK(t, err)
-		rtest.Equals(t, c.expect+"\n", buf.String())
+		rtest.Equals(t, expect+"\n", buf.String())
 
 		// Sanity check: output must be valid JSON.
 		var v interface{}
@@ -90,3 +106,54 @@ func TestLsNodeJSON(t *testing.T) {
 		rtest.OK(t, err)
 	}
 }
+
+func TestLsNcduNode(t *testing.T) {
+	for i, expect := range []string{
+		`{"name":"baz","asize":12345,"dsize":12345,"dev":0,"ino":0,"nlink":1,"notreg":false,"uid":10000000,"gid":20000000,"mode":0,"mtime":-62135596800}`,
+		`{"name":"empty","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":3840,"notreg":false,"uid":1001,"gid":1001,"mode":0,"mtime":-62135596800}`,
+		`{"name":"link","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":true,"uid":0,"gid":0,"mode":511,"mtime":-62135596800}`,
+		`{"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":493,"mtime":1577934245}`,
+		`{"name":"sticky","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":4077,"mtime":-62135596800}`,
+	} {
+		c := lsTestNodes[i]
+		out, err := lsNcduNode(c.path, &c.Node)
+		rtest.OK(t, err)
+		rtest.Equals(t, expect, string(out))
+
+		// Sanity check: output must be valid JSON.
+		var v interface{}
+		err = json.Unmarshal(out, &v)
+		rtest.OK(t, err)
+	}
+}
+
+func TestLsNcdu(t *testing.T) {
+	var buf bytes.Buffer
+	printer := &ncduLsPrinter{
+		out: &buf,
+	}
+
+	printer.Snapshot(&restic.Snapshot{
+		Hostname: "host",
+		Paths:    []string{"/example"},
+	})
+	printer.Node("/directory", &restic.Node{
+		Type: "dir",
+		Name: "directory",
+	})
+	printer.Node("/directory/data", &restic.Node{
+		Type: "file",
+		Name: "data",
+		Size: 42,
+	})
+	printer.LeaveDir("/directory")
+	printer.Close()
+
+	rtest.Equals(t, `[1, 2, {"time":"0001-01-01T00:00:00Z","tree":null,"paths":["/example"],"hostname":"host"},
+  [
+    {"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800},
+    {"name":"data","asize":42,"dsize":42,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800}
+  ]
+]
+`, buf.String())
+}

cmd/restic/cmd_migrate.go

@@ -117,16 +117,11 @@ func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptio
 }
 
 func runMigrate(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, args []string) error {
-	repo, err := OpenRepository(ctx, gopts)
-	if err != nil {
-		return err
-	}
-
-	lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-	defer unlockRepo(lock)
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
 	if err != nil {
 		return err
 	}
+	defer unlock()
 
 	if len(args) == 0 {
 		return checkMigrations(ctx, repo)

cmd/restic/cmd_mount.go

@@ -113,22 +113,23 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
 		return errors.Fatal("wrong number of parameters")
 	}
 
-	debug.Log("start mount")
-	defer debug.Log("finish mount")
+	mountpoint := args[0]
 
-	repo, err := OpenRepository(ctx, gopts)
-	if err != nil {
+	// Check the existence of the mount point at the earliest stage to
+	// prevent unnecessary computations while opening the repository.
+	if _, err := resticfs.Stat(mountpoint); errors.Is(err, os.ErrNotExist) {
+		Verbosef("Mountpoint %s doesn't exist\n", mountpoint)
 		return err
 	}
 
-	if !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
+	debug.Log("start mount")
+	defer debug.Log("finish mount")
+
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
+	if err != nil {
+		return err
 	}
+	defer unlock()
 
 	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
 	err = repo.LoadIndex(ctx, bar)
@@ -136,12 +137,6 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
 		return err
 	}
 
-	mountpoint := args[0]
-
-	if _, err := resticfs.Stat(mountpoint); errors.Is(err, os.ErrNotExist) {
-		Verbosef("Mountpoint %s doesn't exist\n", mountpoint)
-		return err
-	}
 	mountOptions := []systemFuse.MountOption{
 		systemFuse.ReadOnly(),
 		systemFuse.FSName("restic"),
@@ -157,28 +152,15 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
 		}
 	}
 
-	AddCleanupHandler(func(code int) (int, error) {
-		debug.Log("running umount cleanup handler for mount at %v", mountpoint)
-		err := umount(mountpoint)
-		if err != nil {
-			Warnf("unable to umount (maybe already umounted or still in use?): %v\n", err)
-		}
-		// replace error code of sigint
-		if code == 130 {
-			code = 0
-		}
-		return code, nil
-	})
+	systemFuse.Debug = func(msg interface{}) {
+		debug.Log("fuse: %v", msg)
+	}
 
 	c, err := systemFuse.Mount(mountpoint, mountOptions...)
 	if err != nil {
 		return err
 	}
 
-	systemFuse.Debug = func(msg interface{}) {
-		debug.Log("fuse: %v", msg)
-	}
-
 	cfg := fuse.Config{
 		OwnerIsRoot:   opts.OwnerRoot,
 		Filter:        opts.SnapshotFilter,
@@ -192,15 +174,26 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
 	Printf("When finished, quit with Ctrl-c here or umount the mountpoint.\n")
 
 	debug.Log("serving mount at %v", mountpoint)
-	err = fs.Serve(c, root)
-	if err != nil {
-		return err
+
+	done := make(chan struct{})
+
+	go func() {
+		defer close(done)
+		err = fs.Serve(c, root)
+	}()
+
+	select {
+	case <-ctx.Done():
+		debug.Log("running umount cleanup handler for mount at %v", mountpoint)
+		err := systemFuse.Unmount(mountpoint)
+		if err != nil {
+			Warnf("unable to umount (maybe already umounted or still in use?): %v\n", err)
+		}
+
+		return ErrOK
+	case <-done:
+		// clean shutdown, nothing to do
 	}
 
-	<-c.Ready
-	return c.MountError
-}
-
-func umount(mountpoint string) error {
-	return systemFuse.Unmount(mountpoint)
+	return err
 }

cmd/restic/cmd_mount_integration_test.go

@@ -12,8 +12,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/restic/restic/internal/debug"
-	"github.com/restic/restic/internal/repository"
+	systemFuse "github.com/anacrolix/fuse"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
 )
@@ -67,7 +66,7 @@ func testRunMount(t testing.TB, gopts GlobalOptions, dir string, wg *sync.WaitGr
 func testRunUmount(t testing.TB, dir string) {
 	var err error
 	for i := 0; i < mountWait; i++ {
-		if err = umount(dir); err == nil {
+		if err = systemFuse.Unmount(dir); err == nil {
 			t.Logf("directory %v umounted", dir)
 			return
 		}
@@ -87,12 +86,12 @@ func listSnapshots(t testing.TB, dir string) []string {
 	return names
 }
 
-func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Repository, mountpoint, repodir string, snapshotIDs restic.IDs, expectedSnapshotsInFuseDir int) {
+func checkSnapshots(t testing.TB, gopts GlobalOptions, mountpoint string, snapshotIDs restic.IDs, expectedSnapshotsInFuseDir int) {
 	t.Logf("checking for %d snapshots: %v", len(snapshotIDs), snapshotIDs)
 
 	var wg sync.WaitGroup
 	wg.Add(1)
-	go testRunMount(t, global, mountpoint, &wg)
+	go testRunMount(t, gopts, mountpoint, &wg)
 	waitForMount(t, mountpoint)
 	defer wg.Wait()
 	defer testRunUmount(t, mountpoint)
@@ -101,7 +100,7 @@ func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Reposit
 		t.Fatal(`virtual directory "snapshots" doesn't exist`)
 	}
 
-	ids := listSnapshots(t, repodir)
+	ids := listSnapshots(t, gopts.Repo)
 	t.Logf("found %v snapshots in repo: %v", len(ids), ids)
 
 	namesInSnapshots := listSnapshots(t, mountpoint)
@@ -125,6 +124,10 @@ func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Reposit
 		}
 	}
 
+	_, repo, unlock, err := openWithReadLock(context.TODO(), gopts, false)
+	rtest.OK(t, err)
+	defer unlock()
+
 	for _, id := range snapshotIDs {
 		snapshot, err := restic.LoadSnapshot(context.TODO(), repo, id)
 		rtest.OK(t, err)
@@ -160,11 +163,6 @@ func TestMount(t *testing.T) {
 		t.Skip("Skipping fuse tests")
 	}
 
-	debugEnabled := debug.TestLogToStderr(t)
-	if debugEnabled {
-		defer debug.TestDisableLog(t)
-	}
-
 	env, cleanup := withTestEnvironment(t)
 	// must list snapshots more than once
 	env.gopts.backendTestHook = nil
@@ -172,10 +170,7 @@ func TestMount(t *testing.T) {
 
 	testRunInit(t, env.gopts)
 
-	repo, err := OpenRepository(context.TODO(), env.gopts)
-	rtest.OK(t, err)
-
-	checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, []restic.ID{}, 0)
+	checkSnapshots(t, env.gopts, env.mountpoint, []restic.ID{}, 0)
 
 	rtest.SetupTarTestFixture(t, env.testdata, filepath.Join("testdata", "backup-data.tar.gz"))
 
@@ -185,7 +180,7 @@ func TestMount(t *testing.T) {
 	rtest.Assert(t, len(snapshotIDs) == 1,
 		"expected one snapshot, got %v", snapshotIDs)
 
-	checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, snapshotIDs, 2)
+	checkSnapshots(t, env.gopts, env.mountpoint, snapshotIDs, 2)
 
 	// second backup, implicit incremental
 	testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts)
@@ -193,7 +188,7 @@ func TestMount(t *testing.T) {
 	rtest.Assert(t, len(snapshotIDs) == 2,
 		"expected two snapshots, got %v", snapshotIDs)
 
-	checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, snapshotIDs, 3)
+	checkSnapshots(t, env.gopts, env.mountpoint, snapshotIDs, 3)
 
 	// third backup, explicit incremental
 	bopts := BackupOptions{Parent: snapshotIDs[0].String()}
@@ -202,7 +197,7 @@ func TestMount(t *testing.T) {
 	rtest.Assert(t, len(snapshotIDs) == 3,
 		"expected three snapshots, got %v", snapshotIDs)
 
-	checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, snapshotIDs, 4)
+	checkSnapshots(t, env.gopts, env.mountpoint, snapshotIDs, 4)
 }
 
 func TestMountSameTimestamps(t *testing.T) {
@@ -217,14 +212,11 @@ func TestMountSameTimestamps(t *testing.T) {
 
 	rtest.SetupTarTestFixture(t, env.base, filepath.Join("testdata", "repo-same-timestamps.tar.gz"))
 
-	repo, err := OpenRepository(context.TODO(), env.gopts)
-	rtest.OK(t, err)
-
 	ids := []restic.ID{
 		restic.TestParseID("280303689e5027328889a06d718b729e96a1ce6ae9ef8290bff550459ae611ee"),
 		restic.TestParseID("75ad6cdc0868e082f2596d5ab8705e9f7d87316f5bf5690385eeff8dbe49d9f5"),
 		restic.TestParseID("5fd0d8b2ef0fa5d23e58f1e460188abb0f525c0f0c4af8365a1280c807a80a1b"),
 	}
 
-	checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, ids, 4)
+	checkSnapshots(t, env.gopts, env.mountpoint, ids, 4)
 }

cmd/restic/cmd_options.go

@@ -21,7 +21,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	Hidden:            true,
 	DisableAutoGenTag: true,
-	Run: func(cmd *cobra.Command, args []string) {
+	Run: func(_ *cobra.Command, _ []string) {
 		fmt.Printf("All Extended Options:\n")
 		var maxLen int
 		for _, opt := range options.List() {

cmd/restic/cmd_prune.go

@@ -4,25 +4,20 @@ import (
 	"context"
 	"math"
 	"runtime"
-	"sort"
 	"strconv"
 	"strings"
 
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
-	"github.com/restic/restic/internal/index"
-	"github.com/restic/restic/internal/pack"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/ui"
+	"github.com/restic/restic/internal/ui/progress"
+	"github.com/restic/restic/internal/ui/termstatus"
 
 	"github.com/spf13/cobra"
 )
 
-var errorIndexIncomplete = errors.Fatal("index is not complete")
-var errorPacksMissing = errors.Fatal("packs from index missing in repo")
-var errorSizeNotMatching = errors.Fatal("pack size does not match calculated size from index")
-
 var cmdPrune = &cobra.Command{
 	Use:   "prune [flags]",
 	Short: "Remove unneeded data from the repository",
@@ -36,8 +31,10 @@ EXIT STATUS
 Exit status is 0 if the command was successful, and non-zero if there was any error.
 `,
 	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return runPrune(cmd.Context(), pruneOptions, globalOptions)
+	RunE: func(cmd *cobra.Command, _ []string) error {
+		term, cancel := setupTermstatus()
+		defer cancel()
+		return runPrune(cmd.Context(), pruneOptions, globalOptions, term)
 	},
 }
 
@@ -66,10 +63,10 @@ func init() {
 	f := cmdPrune.Flags()
 	f.BoolVarP(&pruneOptions.DryRun, "dry-run", "n", false, "do not modify the repository, just print what would be done")
 	f.StringVarP(&pruneOptions.UnsafeNoSpaceRecovery, "unsafe-recover-no-free-space", "", "", "UNSAFE, READ THE DOCUMENTATION BEFORE USING! Try to recover a repository stuck with no free space. Do not use without trying out 'prune --max-repack-size 0' first.")
-	addPruneOptions(cmdPrune)
+	addPruneOptions(cmdPrune, &pruneOptions)
 }
 
-func addPruneOptions(c *cobra.Command) {
+func addPruneOptions(c *cobra.Command, pruneOptions *PruneOptions) {
 	f := c.Flags()
 	f.StringVar(&pruneOptions.MaxUnused, "max-unused", "5%", "tolerate given `limit` of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited')")
 	f.StringVar(&pruneOptions.MaxRepackSize, "max-repack-size", "", "maximum `size` to repack (allowed suffixes: k/K, m/M, g/G, t/T)")
@@ -100,7 +97,7 @@ func verifyPruneOptions(opts *PruneOptions) error {
 	// parse MaxUnused either as unlimited, a percentage, or an absolute number of bytes
 	switch {
 	case maxUnused == "unlimited":
-		opts.maxUnusedBytes = func(used uint64) uint64 {
+		opts.maxUnusedBytes = func(_ uint64) uint64 {
 			return math.MaxUint64
 		}
 
@@ -129,7 +126,7 @@ func verifyPruneOptions(opts *PruneOptions) error {
 			return errors.Fatalf("invalid number of bytes %q for --max-unused: %v", opts.MaxUnused, err)
 		}
 
-		opts.maxUnusedBytes = func(used uint64) uint64 {
+		opts.maxUnusedBytes = func(_ uint64) uint64 {
 			return uint64(size)
 		}
 	}
@@ -137,7 +134,7 @@ func verifyPruneOptions(opts *PruneOptions) error {
 	return nil
 }
 
-func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error {
+func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, term *termstatus.Terminal) error {
 	err := verifyPruneOptions(&opts)
 	if err != nil {
 		return err
@@ -147,18 +144,11 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error
 		return errors.Fatal("disabled compression and `--repack-uncompressed` are mutually exclusive")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
 	if err != nil {
 		return err
 	}
-
-	if repo.Backend().Connections() < 2 {
-		return errors.Fatal("prune requires a backend connection limit of at least two")
-	}
-
-	if repo.Config().Version < 2 && opts.RepackUncompressed {
-		return errors.Fatal("compression requires at least repository format version 2")
-	}
+	defer unlock()
 
 	if opts.UnsafeNoSpaceRecovery != "" {
 		repoID := repo.Config().ID
@@ -168,16 +158,10 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error
 		opts.unsafeRecovery = true
 	}
 
-	lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-	defer unlockRepo(lock)
-	if err != nil {
-		return err
-	}
-
-	return runPruneWithRepo(ctx, opts, gopts, repo, restic.NewIDSet())
+	return runPruneWithRepo(ctx, opts, gopts, repo, restic.NewIDSet(), term)
 }
 
-func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet) error {
+func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet, term *termstatus.Terminal) error {
 	// we do not need index updates while pruning!
 	repo.DisableAutoIndexUpdate()
 
@@ -185,24 +169,43 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption
 		Print("warning: running prune without a cache, this may be very slow!\n")
 	}
 
-	Verbosef("loading indexes...\n")
+	printer := newTerminalProgressPrinter(gopts.verbosity, term)
+
+	printer.P("loading indexes...\n")
 	// loading the index before the snapshots is ok, as we use an exclusive lock here
-	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
+	bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
 	err := repo.LoadIndex(ctx, bar)
 	if err != nil {
 		return err
 	}
 
-	plan, stats, err := planPrune(ctx, opts, repo, ignoreSnapshots, gopts.Quiet)
+	popts := repository.PruneOptions{
+		DryRun:         opts.DryRun,
+		UnsafeRecovery: opts.unsafeRecovery,
+
+		MaxUnusedBytes: opts.maxUnusedBytes,
+		MaxRepackBytes: opts.MaxRepackBytes,
+
+		RepackCachableOnly: opts.RepackCachableOnly,
+		RepackSmall:        opts.RepackSmall,
+		RepackUncompressed: opts.RepackUncompressed,
+	}
+
+	plan, err := repository.PlanPrune(ctx, popts, repo, func(ctx context.Context, repo restic.Repository) (usedBlobs restic.CountedBlobSet, err error) {
+		return getUsedBlobs(ctx, repo, ignoreSnapshots, printer)
+	}, printer)
 	if err != nil {
 		return err
 	}
-
-	if opts.DryRun {
-		Verbosef("\nWould have made the following changes:")
+	if ctx.Err() != nil {
+		return ctx.Err()
 	}
 
-	err = printPruneStats(stats)
+	if popts.DryRun {
+		printer.P("\nWould have made the following changes:")
+	}
+
+	err = printPruneStats(printer, plan.Stats())
 	if err != nil {
 		return err
 	}
@@ -210,607 +213,55 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption
 	// Trigger GC to reset garbage collection threshold
 	runtime.GC()
 
-	return doPrune(ctx, opts, gopts, repo, plan)
-}
-
-type pruneStats struct {
-	blobs struct {
-		used      uint
-		duplicate uint
-		unused    uint
-		remove    uint
-		repack    uint
-		repackrm  uint
-	}
-	size struct {
-		used         uint64
-		duplicate    uint64
-		unused       uint64
-		remove       uint64
-		repack       uint64
-		repackrm     uint64
-		unref        uint64
-		uncompressed uint64
-	}
-	packs struct {
-		used       uint
-		unused     uint
-		partlyUsed uint
-		unref      uint
-		keep       uint
-		repack     uint
-		remove     uint
-	}
-}
-
-type prunePlan struct {
-	removePacksFirst restic.IDSet          // packs to remove first (unreferenced packs)
-	repackPacks      restic.IDSet          // packs to repack
-	keepBlobs        restic.CountedBlobSet // blobs to keep during repacking
-	removePacks      restic.IDSet          // packs to remove
-	ignorePacks      restic.IDSet          // packs to ignore when rebuilding the index
-}
-
-type packInfo struct {
-	usedBlobs    uint
-	unusedBlobs  uint
-	usedSize     uint64
-	unusedSize   uint64
-	tpe          restic.BlobType
-	uncompressed bool
-}
-
-type packInfoWithID struct {
-	ID restic.ID
-	packInfo
-	mustCompress bool
-}
-
-// planPrune selects which files to rewrite and which to delete and which blobs to keep.
-// Also some summary statistics are returned.
-func planPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (prunePlan, pruneStats, error) {
-	var stats pruneStats
-
-	usedBlobs, err := getUsedBlobs(ctx, repo, ignoreSnapshots, quiet)
-	if err != nil {
-		return prunePlan{}, stats, err
-	}
-
-	Verbosef("searching used packs...\n")
-	keepBlobs, indexPack, err := packInfoFromIndex(ctx, repo.Index(), usedBlobs, &stats)
-	if err != nil {
-		return prunePlan{}, stats, err
-	}
-
-	Verbosef("collecting packs for deletion and repacking\n")
-	plan, err := decidePackAction(ctx, opts, repo, indexPack, &stats, quiet)
-	if err != nil {
-		return prunePlan{}, stats, err
-	}
-
-	if len(plan.repackPacks) != 0 {
-		blobCount := keepBlobs.Len()
-		// when repacking, we do not want to keep blobs which are
-		// already contained in kept packs, so delete them from keepBlobs
-		repo.Index().Each(ctx, func(blob restic.PackedBlob) {
-			if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) {
-				return
-			}
-			keepBlobs.Delete(blob.BlobHandle)
-		})
-
-		if keepBlobs.Len() < blobCount/2 {
-			// replace with copy to shrink map to necessary size if there's a chance to benefit
-			keepBlobs = keepBlobs.Copy()
-		}
-	} else {
-		// keepBlobs is only needed if packs are repacked
-		keepBlobs = nil
-	}
-	plan.keepBlobs = keepBlobs
-
-	return plan, stats, nil
-}
-
-func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs restic.CountedBlobSet, stats *pruneStats) (restic.CountedBlobSet, map[restic.ID]packInfo, error) {
-	// iterate over all blobs in index to find out which blobs are duplicates
-	// The counter in usedBlobs describes how many instances of the blob exist in the repository index
-	// Thus 0 == blob is missing, 1 == blob exists once, >= 2 == duplicates exist
-	idx.Each(ctx, func(blob restic.PackedBlob) {
-		bh := blob.BlobHandle
-		count, ok := usedBlobs[bh]
-		if ok {
-			if count < math.MaxUint8 {
-				// don't overflow, but saturate count at 255
-				// this can lead to a non-optimal pack selection, but won't cause
-				// problems otherwise
-				count++
-			}
-
-			usedBlobs[bh] = count
-		}
-	})
-
-	// Check if all used blobs have been found in index
-	missingBlobs := restic.NewBlobSet()
-	for bh, count := range usedBlobs {
-		if count == 0 {
-			// blob does not exist in any pack files
-			missingBlobs.Insert(bh)
-		}
-	}
-
-	if len(missingBlobs) != 0 {
-		Warnf("%v not found in the index\n\n"+
-			"Integrity check failed: Data seems to be missing.\n"+
-			"Will not start prune to prevent (additional) data loss!\n"+
-			"Please report this error (along with the output of the 'prune' run) at\n"+
-			"https://github.com/restic/restic/issues/new/choose\n", missingBlobs)
-		return nil, nil, errorIndexIncomplete
-	}
-
-	indexPack := make(map[restic.ID]packInfo)
-
-	// save computed pack header size
-	for pid, hdrSize := range pack.Size(ctx, idx, true) {
-		// initialize tpe with NumBlobTypes to indicate it's not set
-		indexPack[pid] = packInfo{tpe: restic.NumBlobTypes, usedSize: uint64(hdrSize)}
-	}
-
-	hasDuplicates := false
-	// iterate over all blobs in index to generate packInfo
-	idx.Each(ctx, func(blob restic.PackedBlob) {
-		ip := indexPack[blob.PackID]
-
-		// Set blob type if not yet set
-		if ip.tpe == restic.NumBlobTypes {
-			ip.tpe = blob.Type
-		}
-
-		// mark mixed packs with "Invalid blob type"
-		if ip.tpe != blob.Type {
-			ip.tpe = restic.InvalidBlob
-		}
-
-		bh := blob.BlobHandle
-		size := uint64(blob.Length)
-		dupCount := usedBlobs[bh]
-		switch {
-		case dupCount >= 2:
-			hasDuplicates = true
-			// mark as unused for now, we will later on select one copy
-			ip.unusedSize += size
-			ip.unusedBlobs++
-
-			// count as duplicate, will later on change one copy to be counted as used
-			stats.size.duplicate += size
-			stats.blobs.duplicate++
-		case dupCount == 1: // used blob, not duplicate
-			ip.usedSize += size
-			ip.usedBlobs++
-
-			stats.size.used += size
-			stats.blobs.used++
-		default: // unused blob
-			ip.unusedSize += size
-			ip.unusedBlobs++
-
-			stats.size.unused += size
-			stats.blobs.unused++
-		}
-		if !blob.IsCompressed() {
-			ip.uncompressed = true
-		}
-		// update indexPack
-		indexPack[blob.PackID] = ip
-	})
-
-	// if duplicate blobs exist, those will be set to either "used" or "unused":
-	// - mark only one occurence of duplicate blobs as used
-	// - if there are already some used blobs in a pack, possibly mark duplicates in this pack as "used"
-	// - if there are no used blobs in a pack, possibly mark duplicates as "unused"
-	if hasDuplicates {
-		// iterate again over all blobs in index (this is pretty cheap, all in-mem)
-		idx.Each(ctx, func(blob restic.PackedBlob) {
-			bh := blob.BlobHandle
-			count, ok := usedBlobs[bh]
-			// skip non-duplicate, aka. normal blobs
-			// count == 0 is used to mark that this was a duplicate blob with only a single occurence remaining
-			if !ok || count == 1 {
-				return
-			}
-
-			ip := indexPack[blob.PackID]
-			size := uint64(blob.Length)
-			switch {
-			case ip.usedBlobs > 0, count == 0:
-				// other used blobs in pack or "last" occurence ->  transition to used
-				ip.usedSize += size
-				ip.usedBlobs++
-				ip.unusedSize -= size
-				ip.unusedBlobs--
-				// same for the global statistics
-				stats.size.used += size
-				stats.blobs.used++
-				stats.size.duplicate -= size
-				stats.blobs.duplicate--
-				// let other occurences remain marked as unused
-				usedBlobs[bh] = 1
-			default:
-				// remain unused and decrease counter
-				count--
-				if count == 1 {
-					// setting count to 1 would lead to forgetting that this blob had duplicates
-					// thus use the special value zero. This will select the last instance of the blob for keeping.
-					count = 0
-				}
-				usedBlobs[bh] = count
-			}
-			// update indexPack
-			indexPack[blob.PackID] = ip
-		})
-	}
-
-	// Sanity check. If no duplicates exist, all blobs have value 1. After handling
-	// duplicates, this also applies to duplicates.
-	for _, count := range usedBlobs {
-		if count != 1 {
-			panic("internal error during blob selection")
-		}
-	}
-
-	return usedBlobs, indexPack, nil
-}
-
-func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Repository, indexPack map[restic.ID]packInfo, stats *pruneStats, quiet bool) (prunePlan, error) {
-	removePacksFirst := restic.NewIDSet()
-	removePacks := restic.NewIDSet()
-	repackPacks := restic.NewIDSet()
-
-	var repackCandidates []packInfoWithID
-	var repackSmallCandidates []packInfoWithID
-	repoVersion := repo.Config().Version
-	// only repack very small files by default
-	targetPackSize := repo.PackSize() / 25
-	if opts.RepackSmall {
-		// consider files with at least 80% of the target size as large enough
-		targetPackSize = repo.PackSize() / 5 * 4
-	}
-
-	// loop over all packs and decide what to do
-	bar := newProgressMax(!quiet, uint64(len(indexPack)), "packs processed")
-	err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
-		p, ok := indexPack[id]
-		if !ok {
-			// Pack was not referenced in index and is not used  => immediately remove!
-			Verboseff("will remove pack %v as it is unused and not indexed\n", id.Str())
-			removePacksFirst.Insert(id)
-			stats.size.unref += uint64(packSize)
-			return nil
-		}
-
-		if p.unusedSize+p.usedSize != uint64(packSize) && p.usedBlobs != 0 {
-			// Pack size does not fit and pack is needed => error
-			// If the pack is not needed, this is no error, the pack can
-			// and will be simply removed, see below.
-			Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic repair index'.\n",
-				id.Str(), p.unusedSize+p.usedSize, packSize)
-			return errorSizeNotMatching
-		}
-
-		// statistics
-		switch {
-		case p.usedBlobs == 0:
-			stats.packs.unused++
-		case p.unusedBlobs == 0:
-			stats.packs.used++
-		default:
-			stats.packs.partlyUsed++
-		}
-
-		if p.uncompressed {
-			stats.size.uncompressed += p.unusedSize + p.usedSize
-		}
-		mustCompress := false
-		if repoVersion >= 2 {
-			// repo v2: always repack tree blobs if uncompressed
-			// compress data blobs if requested
-			mustCompress = (p.tpe == restic.TreeBlob || opts.RepackUncompressed) && p.uncompressed
-		}
-
-		// decide what to do
-		switch {
-		case p.usedBlobs == 0:
-			// All blobs in pack are no longer used => remove pack!
-			removePacks.Insert(id)
-			stats.blobs.remove += p.unusedBlobs
-			stats.size.remove += p.unusedSize
-
-		case opts.RepackCachableOnly && p.tpe == restic.DataBlob:
-			// if this is a data pack and --repack-cacheable-only is set => keep pack!
-			stats.packs.keep++
-
-		case p.unusedBlobs == 0 && p.tpe != restic.InvalidBlob && !mustCompress:
-			if packSize >= int64(targetPackSize) {
-				// All blobs in pack are used and not mixed => keep pack!
-				stats.packs.keep++
-			} else {
-				repackSmallCandidates = append(repackSmallCandidates, packInfoWithID{ID: id, packInfo: p, mustCompress: mustCompress})
-			}
-
-		default:
-			// all other packs are candidates for repacking
-			repackCandidates = append(repackCandidates, packInfoWithID{ID: id, packInfo: p, mustCompress: mustCompress})
-		}
-
-		delete(indexPack, id)
-		bar.Add(1)
-		return nil
-	})
-	bar.Done()
-	if err != nil {
-		return prunePlan{}, err
-	}
-
-	// At this point indexPacks contains only missing packs!
-
-	// missing packs that are not needed can be ignored
-	ignorePacks := restic.NewIDSet()
-	for id, p := range indexPack {
-		if p.usedBlobs == 0 {
-			ignorePacks.Insert(id)
-			stats.blobs.remove += p.unusedBlobs
-			stats.size.remove += p.unusedSize
-			delete(indexPack, id)
-		}
-	}
-
-	if len(indexPack) != 0 {
-		Warnf("The index references %d needed pack files which are missing from the repository:\n", len(indexPack))
-		for id := range indexPack {
-			Warnf("  %v\n", id)
-		}
-		return prunePlan{}, errorPacksMissing
-	}
-	if len(ignorePacks) != 0 {
-		Warnf("Missing but unneeded pack files are referenced in the index, will be repaired\n")
-		for id := range ignorePacks {
-			Warnf("will forget missing pack file %v\n", id)
-		}
-	}
-
-	if len(repackSmallCandidates) < 10 {
-		// too few small files to be worth the trouble, this also prevents endlessly repacking
-		// if there is just a single pack file below the target size
-		stats.packs.keep += uint(len(repackSmallCandidates))
-	} else {
-		repackCandidates = append(repackCandidates, repackSmallCandidates...)
-	}
-
-	// Sort repackCandidates such that packs with highest ratio unused/used space are picked first.
-	// This is equivalent to sorting by unused / total space.
-	// Instead of unused[i] / used[i] > unused[j] / used[j] we use
-	// unused[i] * used[j] > unused[j] * used[i] as uint32*uint32 < uint64
-	// Moreover packs containing trees and too small packs are sorted to the beginning
-	sort.Slice(repackCandidates, func(i, j int) bool {
-		pi := repackCandidates[i].packInfo
-		pj := repackCandidates[j].packInfo
-		switch {
-		case pi.tpe != restic.DataBlob && pj.tpe == restic.DataBlob:
-			return true
-		case pj.tpe != restic.DataBlob && pi.tpe == restic.DataBlob:
-			return false
-		case pi.unusedSize+pi.usedSize < uint64(targetPackSize) && pj.unusedSize+pj.usedSize >= uint64(targetPackSize):
-			return true
-		case pj.unusedSize+pj.usedSize < uint64(targetPackSize) && pi.unusedSize+pi.usedSize >= uint64(targetPackSize):
-			return false
-		}
-		return pi.unusedSize*pj.usedSize > pj.unusedSize*pi.usedSize
-	})
-
-	repack := func(id restic.ID, p packInfo) {
-		repackPacks.Insert(id)
-		stats.blobs.repack += p.unusedBlobs + p.usedBlobs
-		stats.size.repack += p.unusedSize + p.usedSize
-		stats.blobs.repackrm += p.unusedBlobs
-		stats.size.repackrm += p.unusedSize
-		if p.uncompressed {
-			stats.size.uncompressed -= p.unusedSize + p.usedSize
-		}
-	}
-
-	// calculate limit for number of unused bytes in the repo after repacking
-	maxUnusedSizeAfter := opts.maxUnusedBytes(stats.size.used)
-
-	for _, p := range repackCandidates {
-		reachedUnusedSizeAfter := (stats.size.unused-stats.size.remove-stats.size.repackrm < maxUnusedSizeAfter)
-		reachedRepackSize := stats.size.repack+p.unusedSize+p.usedSize >= opts.MaxRepackBytes
-		packIsLargeEnough := p.unusedSize+p.usedSize >= uint64(targetPackSize)
-
-		switch {
-		case reachedRepackSize:
-			stats.packs.keep++
-
-		case p.tpe != restic.DataBlob, p.mustCompress:
-			// repacking non-data packs / uncompressed-trees is only limited by repackSize
-			repack(p.ID, p.packInfo)
-
-		case reachedUnusedSizeAfter && packIsLargeEnough:
-			// for all other packs stop repacking if tolerated unused size is reached.
-			stats.packs.keep++
-
-		default:
-			repack(p.ID, p.packInfo)
-		}
-	}
-
-	stats.packs.unref = uint(len(removePacksFirst))
-	stats.packs.repack = uint(len(repackPacks))
-	stats.packs.remove = uint(len(removePacks))
-
-	if repo.Config().Version < 2 {
-		// compression not supported for repository format version 1
-		stats.size.uncompressed = 0
-	}
-
-	return prunePlan{removePacksFirst: removePacksFirst,
-		removePacks: removePacks,
-		repackPacks: repackPacks,
-		ignorePacks: ignorePacks,
-	}, nil
+	return plan.Execute(ctx, printer)
 }
 
 // printPruneStats prints out the statistics
-func printPruneStats(stats pruneStats) error {
-	Verboseff("\nused:         %10d blobs / %s\n", stats.blobs.used, ui.FormatBytes(stats.size.used))
-	if stats.blobs.duplicate > 0 {
-		Verboseff("duplicates:   %10d blobs / %s\n", stats.blobs.duplicate, ui.FormatBytes(stats.size.duplicate))
+func printPruneStats(printer progress.Printer, stats repository.PruneStats) error {
+	printer.V("\nused:         %10d blobs / %s\n", stats.Blobs.Used, ui.FormatBytes(stats.Size.Used))
+	if stats.Blobs.Duplicate > 0 {
+		printer.V("duplicates:   %10d blobs / %s\n", stats.Blobs.Duplicate, ui.FormatBytes(stats.Size.Duplicate))
 	}
-	Verboseff("unused:       %10d blobs / %s\n", stats.blobs.unused, ui.FormatBytes(stats.size.unused))
-	if stats.size.unref > 0 {
-		Verboseff("unreferenced:                    %s\n", ui.FormatBytes(stats.size.unref))
+	printer.V("unused:       %10d blobs / %s\n", stats.Blobs.Unused, ui.FormatBytes(stats.Size.Unused))
+	if stats.Size.Unref > 0 {
+		printer.V("unreferenced:                    %s\n", ui.FormatBytes(stats.Size.Unref))
 	}
-	totalBlobs := stats.blobs.used + stats.blobs.unused + stats.blobs.duplicate
-	totalSize := stats.size.used + stats.size.duplicate + stats.size.unused + stats.size.unref
-	unusedSize := stats.size.duplicate + stats.size.unused
-	Verboseff("total:        %10d blobs / %s\n", totalBlobs, ui.FormatBytes(totalSize))
-	Verboseff("unused size: %s of total size\n", ui.FormatPercent(unusedSize, totalSize))
+	totalBlobs := stats.Blobs.Used + stats.Blobs.Unused + stats.Blobs.Duplicate
+	totalSize := stats.Size.Used + stats.Size.Duplicate + stats.Size.Unused + stats.Size.Unref
+	unusedSize := stats.Size.Duplicate + stats.Size.Unused
+	printer.V("total:        %10d blobs / %s\n", totalBlobs, ui.FormatBytes(totalSize))
+	printer.V("unused size: %s of total size\n", ui.FormatPercent(unusedSize, totalSize))
 
-	Verbosef("\nto repack:    %10d blobs / %s\n", stats.blobs.repack, ui.FormatBytes(stats.size.repack))
-	Verbosef("this removes: %10d blobs / %s\n", stats.blobs.repackrm, ui.FormatBytes(stats.size.repackrm))
-	Verbosef("to delete:    %10d blobs / %s\n", stats.blobs.remove, ui.FormatBytes(stats.size.remove+stats.size.unref))
-	totalPruneSize := stats.size.remove + stats.size.repackrm + stats.size.unref
-	Verbosef("total prune:  %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, ui.FormatBytes(totalPruneSize))
-	if stats.size.uncompressed > 0 {
-		Verbosef("not yet compressed:              %s\n", ui.FormatBytes(stats.size.uncompressed))
+	printer.P("\nto repack:    %10d blobs / %s\n", stats.Blobs.Repack, ui.FormatBytes(stats.Size.Repack))
+	printer.P("this removes: %10d blobs / %s\n", stats.Blobs.Repackrm, ui.FormatBytes(stats.Size.Repackrm))
+	printer.P("to delete:    %10d blobs / %s\n", stats.Blobs.Remove, ui.FormatBytes(stats.Size.Remove+stats.Size.Unref))
+	totalPruneSize := stats.Size.Remove + stats.Size.Repackrm + stats.Size.Unref
+	printer.P("total prune:  %10d blobs / %s\n", stats.Blobs.Remove+stats.Blobs.Repackrm, ui.FormatBytes(totalPruneSize))
+	if stats.Size.Uncompressed > 0 {
+		printer.P("not yet compressed:              %s\n", ui.FormatBytes(stats.Size.Uncompressed))
 	}
-	Verbosef("remaining:    %10d blobs / %s\n", totalBlobs-(stats.blobs.remove+stats.blobs.repackrm), ui.FormatBytes(totalSize-totalPruneSize))
-	unusedAfter := unusedSize - stats.size.remove - stats.size.repackrm
-	Verbosef("unused size after prune: %s (%s of remaining size)\n",
+	printer.P("remaining:    %10d blobs / %s\n", totalBlobs-(stats.Blobs.Remove+stats.Blobs.Repackrm), ui.FormatBytes(totalSize-totalPruneSize))
+	unusedAfter := unusedSize - stats.Size.Remove - stats.Size.Repackrm
+	printer.P("unused size after prune: %s (%s of remaining size)\n",
 		ui.FormatBytes(unusedAfter), ui.FormatPercent(unusedAfter, totalSize-totalPruneSize))
-	Verbosef("\n")
-	Verboseff("totally used packs: %10d\n", stats.packs.used)
-	Verboseff("partly used packs:  %10d\n", stats.packs.partlyUsed)
-	Verboseff("unused packs:       %10d\n\n", stats.packs.unused)
+	printer.P("\n")
+	printer.V("totally used packs: %10d\n", stats.Packs.Used)
+	printer.V("partly used packs:  %10d\n", stats.Packs.PartlyUsed)
+	printer.V("unused packs:       %10d\n\n", stats.Packs.Unused)
 
-	Verboseff("to keep:      %10d packs\n", stats.packs.keep)
-	Verboseff("to repack:    %10d packs\n", stats.packs.repack)
-	Verboseff("to delete:    %10d packs\n", stats.packs.remove)
-	if stats.packs.unref > 0 {
-		Verboseff("to delete:    %10d unreferenced packs\n\n", stats.packs.unref)
+	printer.V("to keep:      %10d packs\n", stats.Packs.Keep)
+	printer.V("to repack:    %10d packs\n", stats.Packs.Repack)
+	printer.V("to delete:    %10d packs\n", stats.Packs.Remove)
+	if stats.Packs.Unref > 0 {
+		printer.V("to delete:    %10d unreferenced packs\n\n", stats.Packs.Unref)
 	}
 	return nil
 }
 
-// doPrune does the actual pruning:
-// - remove unreferenced packs first
-// - repack given pack files while keeping the given blobs
-// - rebuild the index while ignoring all files that will be deleted
-// - delete the files
-// plan.removePacks and plan.ignorePacks are modified in this function.
-func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo restic.Repository, plan prunePlan) (err error) {
-	if opts.DryRun {
-		if !gopts.JSON && gopts.verbosity >= 2 {
-			Printf("Repeated prune dry-runs can report slightly different amounts of data to keep or repack. This is expected behavior.\n\n")
-			if len(plan.removePacksFirst) > 0 {
-				Printf("Would have removed the following unreferenced packs:\n%v\n\n", plan.removePacksFirst)
-			}
-			Printf("Would have repacked and removed the following packs:\n%v\n\n", plan.repackPacks)
-			Printf("Would have removed the following no longer used packs:\n%v\n\n", plan.removePacks)
-		}
-		// Always quit here if DryRun was set!
-		return nil
-	}
-
-	// unreferenced packs can be safely deleted first
-	if len(plan.removePacksFirst) != 0 {
-		Verbosef("deleting unreferenced packs\n")
-		DeleteFiles(ctx, gopts, repo, plan.removePacksFirst, restic.PackFile)
-	}
-
-	if len(plan.repackPacks) != 0 {
-		Verbosef("repacking packs\n")
-		bar := newProgressMax(!gopts.Quiet, uint64(len(plan.repackPacks)), "packs repacked")
-		_, err := repository.Repack(ctx, repo, repo, plan.repackPacks, plan.keepBlobs, bar)
-		bar.Done()
-		if err != nil {
-			return errors.Fatal(err.Error())
-		}
-
-		// Also remove repacked packs
-		plan.removePacks.Merge(plan.repackPacks)
-
-		if len(plan.keepBlobs) != 0 {
-			Warnf("%v was not repacked\n\n"+
-				"Integrity check failed.\n"+
-				"Please report this error (along with the output of the 'prune' run) at\n"+
-				"https://github.com/restic/restic/issues/new/choose\n", plan.keepBlobs)
-			return errors.Fatal("internal error: blobs were not repacked")
-		}
-
-		// allow GC of the blob set
-		plan.keepBlobs = nil
-	}
-
-	if len(plan.ignorePacks) == 0 {
-		plan.ignorePacks = plan.removePacks
-	} else {
-		plan.ignorePacks.Merge(plan.removePacks)
-	}
-
-	if opts.unsafeRecovery {
-		Verbosef("deleting index files\n")
-		indexFiles := repo.Index().(*index.MasterIndex).IDs()
-		err = DeleteFilesChecked(ctx, gopts, repo, indexFiles, restic.IndexFile)
-		if err != nil {
-			return errors.Fatalf("%s", err)
-		}
-	} else if len(plan.ignorePacks) != 0 {
-		err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil)
-		if err != nil {
-			return errors.Fatalf("%s", err)
-		}
-	}
-
-	if len(plan.removePacks) != 0 {
-		Verbosef("removing %d old packs\n", len(plan.removePacks))
-		DeleteFiles(ctx, gopts, repo, plan.removePacks, restic.PackFile)
-	}
-
-	if opts.unsafeRecovery {
-		_, err = writeIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil)
-		if err != nil {
-			return errors.Fatalf("%s", err)
-		}
-	}
-
-	Verbosef("done\n")
-	return nil
-}
-
-func writeIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) (restic.IDSet, error) {
-	Verbosef("rebuilding index\n")
-
-	bar := newProgressMax(!gopts.Quiet, 0, "packs processed")
-	obsoleteIndexes, err := repo.Index().Save(ctx, repo, removePacks, extraObsolete, bar)
-	bar.Done()
-	return obsoleteIndexes, err
-}
-
-func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error {
-	obsoleteIndexes, err := writeIndexFiles(ctx, gopts, repo, removePacks, extraObsolete)
-	if err != nil {
-		return err
-	}
-
-	Verbosef("deleting obsolete index files\n")
-	return DeleteFilesChecked(ctx, gopts, repo, obsoleteIndexes, restic.IndexFile)
-}
-
-func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (usedBlobs restic.CountedBlobSet, err error) {
+func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, printer progress.Printer) (usedBlobs restic.CountedBlobSet, err error) {
 	var snapshotTrees restic.IDs
-	Verbosef("loading all snapshots...\n")
-	err = restic.ForAllSnapshots(ctx, repo.Backend(), repo, ignoreSnapshots,
+	printer.P("loading all snapshots...\n")
+	err = restic.ForAllSnapshots(ctx, repo, repo, ignoreSnapshots,
 		func(id restic.ID, sn *restic.Snapshot, err error) error {
 			if err != nil {
 				debug.Log("failed to load snapshot %v (error %v)", id, err)
@@ -824,19 +275,16 @@ func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots r
 		return nil, errors.Fatalf("failed loading snapshot: %v", err)
 	}
 
-	Verbosef("finding data that is still in use for %d snapshots\n", len(snapshotTrees))
+	printer.P("finding data that is still in use for %d snapshots\n", len(snapshotTrees))
 
 	usedBlobs = restic.NewCountedBlobSet()
 
-	bar := newProgressMax(!quiet, uint64(len(snapshotTrees)), "snapshots")
+	bar := printer.NewCounter("snapshots")
+	bar.SetMax(uint64(len(snapshotTrees)))
 	defer bar.Done()
 
 	err = restic.FindUsedBlobs(ctx, repo, snapshotTrees, usedBlobs, bar)
 	if err != nil {
-		if repo.Backend().IsNotExist(err) {
-			return nil, errors.Fatal("unable to load a tree from the repository: " + err.Error())
-		}
-
 		return nil, err
 	}
 	return usedBlobs, nil

cmd/restic/cmd_prune_integration_test.go

@@ -6,17 +6,21 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/backend"
+	"github.com/restic/restic/internal/repository"
 	rtest "github.com/restic/restic/internal/test"
+	"github.com/restic/restic/internal/ui/termstatus"
 )
 
 func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
 	oldHook := gopts.backendTestHook
-	gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil }
+	gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil }
 	defer func() {
 		gopts.backendTestHook = oldHook
 	}()
-	rtest.OK(t, runPrune(context.TODO(), opts, gopts))
+	rtest.OK(t, withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+		return runPrune(context.TODO(), opts, gopts, term)
+	}))
 }
 
 func TestPrune(t *testing.T) {
@@ -31,7 +35,7 @@ func testPruneVariants(t *testing.T, unsafeNoSpaceRecovery bool) {
 	}
 	t.Run("0"+suffix, func(t *testing.T) {
 		opts := PruneOptions{MaxUnused: "0%", unsafeRecovery: unsafeNoSpaceRecovery}
-		checkOpts := CheckOptions{ReadData: true, CheckUnused: true}
+		checkOpts := CheckOptions{ReadData: true, CheckUnused: !unsafeNoSpaceRecovery}
 		testPrune(t, opts, checkOpts)
 	})
 
@@ -81,7 +85,12 @@ func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) {
 			DryRun: true,
 			Last:   1,
 		}
-		return runForget(context.TODO(), opts, gopts, args)
+		pruneOpts := PruneOptions{
+			MaxUnused: "5%",
+		}
+		return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+			return runForget(context.TODO(), opts, pruneOpts, gopts, term, args)
+		})
 	})
 	rtest.OK(t, err)
 
@@ -130,12 +139,14 @@ func TestPruneWithDamagedRepository(t *testing.T) {
 	removePacksExcept(env.gopts, t, oldPacks, false)
 
 	oldHook := env.gopts.backendTestHook
-	env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil }
+	env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil }
 	defer func() {
 		env.gopts.backendTestHook = oldHook
 	}()
 	// prune should fail
-	rtest.Assert(t, runPrune(context.TODO(), pruneDefaultOptions, env.gopts) == errorPacksMissing,
+	rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+		return runPrune(context.TODO(), pruneDefaultOptions, env.gopts, term)
+	}) == repository.ErrPacksMissing,
 		"prune should have reported index not complete error")
 }
 
@@ -215,7 +226,9 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
 		testRunPrune(t, env.gopts, optionsPrune)
 		testRunCheck(t, env.gopts)
 	} else {
-		rtest.Assert(t, runPrune(context.TODO(), optionsPrune, env.gopts) != nil,
+		rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+			return runPrune(context.TODO(), optionsPrune, env.gopts, term)
+		}) != nil,
 			"prune should have reported an error")
 	}
 }

cmd/restic/cmd_recover.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"time"
 
-	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 	"github.com/spf13/cobra"
@@ -26,7 +25,7 @@ EXIT STATUS
 Exit status is 0 if the command was successful, and non-zero if there was any error.
 `,
 	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
+	RunE: func(cmd *cobra.Command, _ []string) error {
 		return runRecover(cmd.Context(), globalOptions)
 	},
 }
@@ -41,18 +40,13 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 		return err
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, false)
 	if err != nil {
 		return err
 	}
+	defer unlock()
 
-	lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-	defer unlockRepo(lock)
-	if err != nil {
-		return err
-	}
-
-	snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
+	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
@@ -67,16 +61,22 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 	// tree. If it is not referenced, we have a root tree.
 	trees := make(map[restic.ID]bool)
 
-	repo.Index().Each(ctx, func(blob restic.PackedBlob) {
+	err = repo.Index().Each(ctx, func(blob restic.PackedBlob) {
 		if blob.Type == restic.TreeBlob {
 			trees[blob.Blob.ID] = false
 		}
 	})
+	if err != nil {
+		return err
+	}
 
 	Verbosef("load %d trees\n", len(trees))
 	bar = newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
 	for id := range trees {
 		tree, err := restic.LoadTree(ctx, repo, id)
+		if ctx.Err() != nil {
+			return ctx.Err()
+		}
 		if err != nil {
 			Warnf("unable to load tree %v: %v\n", id.Str(), err)
 			continue
@@ -92,7 +92,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 	bar.Done()
 
 	Verbosef("load snapshots\n")
-	err = restic.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error {
+	err = restic.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(_ restic.ID, sn *restic.Snapshot, _ error) error {
 		trees[*sn.Tree] = true
 		return nil
 	})
@@ -159,7 +159,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
 
 }
 
-func createSnapshot(ctx context.Context, name, hostname string, tags []string, repo restic.Repository, tree *restic.ID) error {
+func createSnapshot(ctx context.Context, name, hostname string, tags []string, repo restic.SaverUnpacked, tree *restic.ID) error {
 	sn, err := restic.NewSnapshot([]string{name}, tags, hostname, time.Now())
 	if err != nil {
 		return errors.Fatalf("unable to save snapshot: %v", err)

cmd/restic/cmd_repair_index.go

@@ -3,10 +3,8 @@ package main
 import (
 	"context"
 
-	"github.com/restic/restic/internal/index"
-	"github.com/restic/restic/internal/pack"
 	"github.com/restic/restic/internal/repository"
-	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui/termstatus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 )
@@ -24,8 +22,10 @@ EXIT STATUS
 Exit status is 0 if the command was successful, and non-zero if there was any error.
 `,
 	DisableAutoGenTag: true,
-	RunE: func(cmd *cobra.Command, args []string) error {
-		return runRebuildIndex(cmd.Context(), repairIndexOptions, globalOptions)
+	RunE: func(cmd *cobra.Command, _ []string) error {
+		term, cancel := setupTermstatus()
+		defer cancel()
+		return runRebuildIndex(cmd.Context(), repairIndexOptions, globalOptions, term)
 	},
 }
 
@@ -55,110 +55,22 @@ func init() {
 	}
 }
 
-func runRebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions) error {
-	repo, err := OpenRepository(ctx, gopts)
+func runRebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions, term *termstatus.Terminal) error {
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
+	if err != nil {
+		return err
+	}
+	defer unlock()
+
+	printer := newTerminalProgressPrinter(gopts.verbosity, term)
+
+	err = repository.RepairIndex(ctx, repo, repository.RepairIndexOptions{
+		ReadAllPacks: opts.ReadAllPacks,
+	}, printer)
 	if err != nil {
 		return err
 	}
 
-	lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-	defer unlockRepo(lock)
-	if err != nil {
-		return err
-	}
-
-	return rebuildIndex(ctx, opts, gopts, repo)
-}
-
-func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions, repo *repository.Repository) error {
-	var obsoleteIndexes restic.IDs
-	packSizeFromList := make(map[restic.ID]int64)
-	packSizeFromIndex := make(map[restic.ID]int64)
-	removePacks := restic.NewIDSet()
-
-	if opts.ReadAllPacks {
-		// get list of old index files but start with empty index
-		err := repo.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error {
-			obsoleteIndexes = append(obsoleteIndexes, id)
-			return nil
-		})
-		if err != nil {
-			return err
-		}
-	} else {
-		Verbosef("loading indexes...\n")
-		mi := index.NewMasterIndex()
-		err := index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
-			if err != nil {
-				Warnf("removing invalid index %v: %v\n", id, err)
-				obsoleteIndexes = append(obsoleteIndexes, id)
-				return nil
-			}
-
-			mi.Insert(idx)
-			return nil
-		})
-		if err != nil {
-			return err
-		}
-
-		err = mi.MergeFinalIndexes()
-		if err != nil {
-			return err
-		}
-
-		err = repo.SetIndex(mi)
-		if err != nil {
-			return err
-		}
-		packSizeFromIndex = pack.Size(ctx, repo.Index(), false)
-	}
-
-	Verbosef("getting pack files to read...\n")
-	err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
-		size, ok := packSizeFromIndex[id]
-		if !ok || size != packSize {
-			// Pack was not referenced in index or size does not match
-			packSizeFromList[id] = packSize
-			removePacks.Insert(id)
-		}
-		if !ok {
-			Warnf("adding pack file to index %v\n", id)
-		} else if size != packSize {
-			Warnf("reindexing pack file %v with unexpected size %v instead of %v\n", id, packSize, size)
-		}
-		delete(packSizeFromIndex, id)
-		return nil
-	})
-	if err != nil {
-		return err
-	}
-	for id := range packSizeFromIndex {
-		// forget pack files that are referenced in the index but do not exist
-		// when rebuilding the index
-		removePacks.Insert(id)
-		Warnf("removing not found pack file %v\n", id)
-	}
-
-	if len(packSizeFromList) > 0 {
-		Verbosef("reading pack files\n")
-		bar := newProgressMax(!gopts.Quiet, uint64(len(packSizeFromList)), "packs")
-		invalidFiles, err := repo.CreateIndexFromPacks(ctx, packSizeFromList, bar)
-		bar.Done()
-		if err != nil {
-			return err
-		}
-
-		for _, id := range invalidFiles {
-			Verboseff("skipped incomplete pack file: %v\n", id)
-		}
-	}
-
-	err = rebuildIndexFiles(ctx, gopts, repo, removePacks, obsoleteIndexes)
-	if err != nil {
-		return err
-	}
-	Verbosef("done\n")
-
+	printer.P("done\n")
 	return nil
 }

cmd/restic/cmd_repair_index_integration_test.go

@@ -8,16 +8,20 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/index"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
+	"github.com/restic/restic/internal/ui/termstatus"
 )
 
 func testRunRebuildIndex(t testing.TB, gopts GlobalOptions) {
 	rtest.OK(t, withRestoreGlobalOptions(func() error {
-		globalOptions.stdout = io.Discard
-		return runRebuildIndex(context.TODO(), RepairIndexOptions{}, gopts)
+		return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+			globalOptions.stdout = io.Discard
+			return runRebuildIndex(context.TODO(), RepairIndexOptions{}, gopts, term)
+		})
 	}))
 }
 
@@ -70,12 +74,12 @@ func TestRebuildIndexAlwaysFull(t *testing.T) {
 
 // indexErrorBackend modifies the first index after reading.
 type indexErrorBackend struct {
-	restic.Backend
+	backend.Backend
 	lock     sync.Mutex
 	hasErred bool
 }
 
-func (b *indexErrorBackend) Load(ctx context.Context, h restic.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
+func (b *indexErrorBackend) Load(ctx context.Context, h backend.Handle, length int, offset int64, consumer func(rd io.Reader) error) error {
 	return b.Backend.Load(ctx, h, length, offset, func(rd io.Reader) error {
 		// protect hasErred
 		b.lock.Lock()
@@ -101,7 +105,7 @@ func (erd errorReadCloser) Read(p []byte) (int, error) {
 }
 
 func TestRebuildIndexDamage(t *testing.T) {
-	testRebuildIndex(t, func(r restic.Backend) (restic.Backend, error) {
+	testRebuildIndex(t, func(r backend.Backend) (backend.Backend, error) {
 		return &indexErrorBackend{
 			Backend: r,
 		}, nil
@@ -109,11 +113,11 @@ func TestRebuildIndexDamage(t *testing.T) {
 }
 
 type appendOnlyBackend struct {
-	restic.Backend
+	backend.Backend
 }
 
 // called via repo.Backend().Remove()
-func (b *appendOnlyBackend) Remove(_ context.Context, h restic.Handle) error {
+func (b *appendOnlyBackend) Remove(_ context.Context, h backend.Handle) error {
 	return errors.Errorf("Failed to remove %v", h)
 }
 
@@ -125,12 +129,13 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) {
 	rtest.SetupTarTestFixture(t, env.base, datafile)
 
 	err := withRestoreGlobalOptions(func() error {
-		globalOptions.stdout = io.Discard
-
-		env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
+		env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
 			return &appendOnlyBackend{r}, nil
 		}
-		return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts)
+		return withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
+			globalOptions.stdout = io.Discard
+			return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts, term)
+		})
 	})
 
 	if err == nil {

cmd/restic/cmd_repair_packs.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"io"
 	"os"
@@ -8,16 +9,14 @@ import (
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui/termstatus"
 	"github.com/spf13/cobra"
-	"golang.org/x/sync/errgroup"
 )
 
 var cmdRepairPacks = &cobra.Command{
 	Use:   "packs [packIDs...]",
 	Short: "Salvage damaged pack files",
 	Long: `
-WARNING: The CLI for this command is experimental and will likely change in the future!
-
 The "repair packs" command extracts intact blobs from the specified pack files, rebuilds
 the index to remove the damaged pack files and removes the pack files from the repository.
 
@@ -28,7 +27,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return runRepairPacks(cmd.Context(), globalOptions, args)
+		term, cancel := setupTermstatus()
+		defer cancel()
+		return runRepairPacks(cmd.Context(), globalOptions, term, args)
 	},
 }
 
@@ -36,14 +37,7 @@ func init() {
 	cmdRepair.AddCommand(cmdRepairPacks)
 }
 
-func runRepairPacks(ctx context.Context, gopts GlobalOptions, args []string) error {
-	// FIXME discuss and add proper feature flag mechanism
-	flag, _ := os.LookupEnv("RESTIC_FEATURES")
-	if flag != "repair-packs-v1" {
-		return errors.Fatal("This command is experimental and may change/be removed without notice between restic versions. " +
-			"Set the environment variable 'RESTIC_FEATURES=repair-packs-v1' to enable it.")
-	}
-
+func runRepairPacks(ctx context.Context, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
 	ids := restic.NewIDSet()
 	for _, arg := range args {
 		id, err := restic.ParseID(arg)
@@ -56,103 +50,46 @@ func runRepairPacks(ctx context.Context, gopts GlobalOptions, args []string) err
 		return errors.Fatal("no ids specified")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
 	if err != nil {
 		return err
 	}
+	defer unlock()
 
-	lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-	defer unlockRepo(lock)
-	if err != nil {
-		return err
-	}
+	printer := newTerminalProgressPrinter(gopts.verbosity, term)
 
-	return repairPacks(ctx, gopts, repo, ids)
-}
-
-func repairPacks(ctx context.Context, gopts GlobalOptions, repo *repository.Repository, ids restic.IDSet) error {
-	bar := newIndexProgress(gopts.Quiet, gopts.JSON)
-	err := repo.LoadIndex(ctx, bar)
+	bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
+	err = repo.LoadIndex(ctx, bar)
 	if err != nil {
 		return errors.Fatalf("%s", err)
 	}
 
-	Warnf("saving backup copies of pack files in current folder\n")
+	printer.P("saving backup copies of pack files to current folder")
 	for id := range ids {
+		buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
+		// corrupted data is fine
+		if buf == nil {
+			return err
+		}
+
 		f, err := os.OpenFile("pack-"+id.String(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o666)
 		if err != nil {
-			return errors.Fatalf("%s", err)
-		}
-
-		err = repo.Backend().Load(ctx, restic.Handle{Type: restic.PackFile, Name: id.String()}, 0, 0, func(rd io.Reader) error {
-			_, err := f.Seek(0, 0)
-			if err != nil {
-				return err
-			}
-			_, err = io.Copy(f, rd)
 			return err
-		})
-		if err != nil {
-			return errors.Fatalf("%s", err)
+		}
+		if _, err := io.Copy(f, bytes.NewReader(buf)); err != nil {
+			_ = f.Close()
+			return err
+		}
+		if err := f.Close(); err != nil {
+			return err
 		}
 	}
 
-	wg, wgCtx := errgroup.WithContext(ctx)
-	repo.StartPackUploader(wgCtx, wg)
-	repo.DisableAutoIndexUpdate()
-
-	Warnf("salvaging intact data from specified pack files\n")
-	bar = newProgressMax(!gopts.Quiet, uint64(len(ids)), "pack files")
-	defer bar.Done()
-
-	wg.Go(func() error {
-		// examine all data the indexes have for the pack file
-		for b := range repo.Index().ListPacks(wgCtx, ids) {
-			blobs := b.Blobs
-			if len(blobs) == 0 {
-				Warnf("no blobs found for pack %v\n", b.PackID)
-				bar.Add(1)
-				continue
-			}
-
-			err = repository.StreamPack(wgCtx, repo.Backend().Load, repo.Key(), b.PackID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error {
-				if err != nil {
-					// Fallback path
-					buf, err = repo.LoadBlob(wgCtx, blob.Type, blob.ID, nil)
-					if err != nil {
-						Warnf("failed to load blob %v: %v\n", blob.ID, err)
-						return nil
-					}
-				}
-				id, _, _, err := repo.SaveBlob(wgCtx, blob.Type, buf, restic.ID{}, true)
-				if !id.Equal(blob.ID) {
-					panic("pack id mismatch during upload")
-				}
-				return err
-			})
-			if err != nil {
-				return err
-			}
-			bar.Add(1)
-		}
-		return repo.Flush(wgCtx)
-	})
-
-	if err := wg.Wait(); err != nil {
-		return errors.Fatalf("%s", err)
-	}
-	bar.Done()
-
-	// remove salvaged packs from index
-	err = rebuildIndexFiles(ctx, gopts, repo, ids, nil)
+	err = repository.RepairPacks(ctx, repo, ids, printer)
 	if err != nil {
 		return errors.Fatalf("%s", err)
 	}
 
-	// cleanup
-	Warnf("removing salvaged pack files\n")
-	DeleteFiles(ctx, gopts, repo, ids, restic.PackFile)
-
 	Warnf("\nUse `restic repair snapshots --forget` to remove the corrupted data blobs from all snapshots\n")
 	return nil
 }

cmd/restic/cmd_repair_snapshots.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 
-	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/restic"
 	"github.com/restic/restic/internal/walker"
@@ -67,24 +66,13 @@ func init() {
 }
 
 func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOptions, args []string) error {
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun)
 	if err != nil {
 		return err
 	}
+	defer unlock()
 
-	if !opts.DryRun {
-		var lock *restic.Lock
-		var err error
-		lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	} else {
-		repo.SetDryRun()
-	}
-
-	snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
+	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
@@ -126,7 +114,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
 			node.Size = newSize
 			return node
 		},
-		RewriteFailedTree: func(nodeID restic.ID, path string, _ error) (restic.ID, error) {
+		RewriteFailedTree: func(_ restic.ID, path string, _ error) (restic.ID, error) {
 			if path == "/" {
 				Verbosef("  dir %q: not readable\n", path)
 				// remove snapshots with invalid root node
@@ -145,11 +133,11 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
 
 	changedCount := 0
 	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) {
-		Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
+		Verbosef("\n%v\n", sn)
 		changed, err := filterAndReplaceSnapshot(ctx, repo, sn,
 			func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
 				return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
-			}, opts.DryRun, opts.Forget, "repaired")
+			}, opts.DryRun, opts.Forget, nil, "repaired")
 		if err != nil {
 			return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err)
 		}
@@ -157,6 +145,9 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
 			changedCount++
 		}
 	}
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 
 	Verbosef("\n")
 	if changedCount == 0 {

cmd/restic/cmd_restore.go

@@ -3,7 +3,6 @@ package main
 import (
 	"context"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/restic/restic/internal/debug"
@@ -38,31 +37,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 `,
 	DisableAutoGenTag: true,
 	RunE: func(cmd *cobra.Command, args []string) error {
-		ctx := cmd.Context()
-		var wg sync.WaitGroup
-		cancelCtx, cancel := context.WithCancel(ctx)
-		defer func() {
-			// shutdown termstatus
-			cancel()
-			wg.Wait()
-		}()
-
-		term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet)
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			term.Run(cancelCtx)
-		}()
-
-		// allow usage of warnf / verbosef
-		prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr
-		defer func() {
-			globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr
-		}()
-		stdioWrapper := ui.NewStdioWrapper(term)
-		globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr()
-
-		return runRestore(ctx, restoreOptions, globalOptions, term, args)
+		term, cancel := setupTermstatus()
+		defer cancel()
+		return runRestore(cmd.Context(), restoreOptions, globalOptions, term, args)
 	},
 }
 
@@ -150,25 +127,17 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
 
 	debug.Log("restore %v to %v", snapshotIDString, opts.Target)
 
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
-
-	if !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
+	defer unlock()
 
 	sn, subfolder, err := (&restic.SnapshotFilter{
 		Hosts: opts.Hosts,
 		Paths: opts.Paths,
 		Tags:  opts.Tags,
-	}).FindLatest(ctx, repo.Backend(), repo, snapshotIDString)
+	}).FindLatest(ctx, repo, repo, snapshotIDString)
 	if err != nil {
 		return errors.Fatalf("failed to find snapshot: %v", err)
 	}
@@ -201,10 +170,13 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
 		totalErrors++
 		return nil
 	}
+	res.Warn = func(message string) {
+		msg.E("Warning: %s\n", message)
+	}
 
 	excludePatterns := filter.ParsePatterns(opts.Exclude)
 	insensitiveExcludePatterns := filter.ParsePatterns(opts.InsensitiveExclude)
-	selectExcludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
+	selectExcludeFilter := func(item string, _ string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
 		matched, err := filter.List(excludePatterns, item)
 		if err != nil {
 			msg.E("error for exclude pattern: %v", err)
@@ -227,7 +199,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions,
 
 	includePatterns := filter.ParsePatterns(opts.Include)
 	insensitiveIncludePatterns := filter.ParsePatterns(opts.InsensitiveInclude)
-	selectIncludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
+	selectIncludeFilter := func(item string, _ string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) {
 		matched, childMayMatch, err := filter.ListWithChild(includePatterns, item)
 		if err != nil {
 			msg.E("error for include pattern: %v", err)

cmd/restic/cmd_restore_integration_test.go

@@ -4,13 +4,14 @@ import (
 	"context"
 	"fmt"
 	"io"
-	mrand "math/rand"
+	"math/rand"
 	"os"
 	"path/filepath"
 	"syscall"
 	"testing"
 	"time"
 
+	"github.com/restic/restic/internal/feature"
 	"github.com/restic/restic/internal/filter"
 	"github.com/restic/restic/internal/restic"
 	rtest "github.com/restic/restic/internal/test"
@@ -116,7 +117,7 @@ func TestRestore(t *testing.T) {
 	for i := 0; i < 10; i++ {
 		p := filepath.Join(env.testdata, fmt.Sprintf("foo/bar/testfile%v", i))
 		rtest.OK(t, os.MkdirAll(filepath.Dir(p), 0755))
-		rtest.OK(t, appendRandomData(p, uint(mrand.Intn(2<<21))))
+		rtest.OK(t, appendRandomData(p, uint(rand.Intn(2<<21))))
 	}
 
 	opts := BackupOptions{}
@@ -274,6 +275,7 @@ func TestRestoreNoMetadataOnIgnoredIntermediateDirs(t *testing.T) {
 }
 
 func TestRestoreLocalLayout(t *testing.T) {
+	defer feature.TestSetFlag(t, feature.Flag, feature.DeprecateS3LegacyLayout, false)()
 	env, cleanup := withTestEnvironment(t)
 	defer cleanup()
 

cmd/restic/cmd_rewrite.go

@@ -3,11 +3,11 @@ package main
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/spf13/cobra"
 	"golang.org/x/sync/errgroup"
 
-	"github.com/restic/restic/internal/backend"
 	"github.com/restic/restic/internal/debug"
 	"github.com/restic/restic/internal/errors"
 	"github.com/restic/restic/internal/repository"
@@ -46,11 +46,42 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
 	},
 }
 
+type snapshotMetadata struct {
+	Hostname string
+	Time     *time.Time
+}
+
+type snapshotMetadataArgs struct {
+	Hostname string
+	Time     string
+}
+
+func (sma snapshotMetadataArgs) empty() bool {
+	return sma.Hostname == "" && sma.Time == ""
+}
+
+func (sma snapshotMetadataArgs) convert() (*snapshotMetadata, error) {
+	if sma.empty() {
+		return nil, nil
+	}
+
+	var timeStamp *time.Time
+	if sma.Time != "" {
+		t, err := time.ParseInLocation(TimeFormat, sma.Time, time.Local)
+		if err != nil {
+			return nil, errors.Fatalf("error in time option: %v\n", err)
+		}
+		timeStamp = &t
+	}
+	return &snapshotMetadata{Hostname: sma.Hostname, Time: timeStamp}, nil
+}
+
 // RewriteOptions collects all options for the rewrite command.
 type RewriteOptions struct {
 	Forget bool
 	DryRun bool
 
+	Metadata snapshotMetadataArgs
 	restic.SnapshotFilter
 	excludePatternOptions
 }
@@ -63,11 +94,15 @@ func init() {
 	f := cmdRewrite.Flags()
 	f.BoolVarP(&rewriteOptions.Forget, "forget", "", false, "remove original snapshots after creating new ones")
 	f.BoolVarP(&rewriteOptions.DryRun, "dry-run", "n", false, "do not do anything, just print what would be done")
+	f.StringVar(&rewriteOptions.Metadata.Hostname, "new-host", "", "replace hostname")
+	f.StringVar(&rewriteOptions.Metadata.Time, "new-time", "", "replace time of the backup")
 
 	initMultiSnapshotFilter(f, &rewriteOptions.SnapshotFilter, true)
 	initExcludePatternOptions(f, &rewriteOptions.excludePatternOptions)
 }
 
+type rewriteFilterFunc func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error)
+
 func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *restic.Snapshot, opts RewriteOptions) (bool, error) {
 	if sn.Tree == nil {
 		return false, errors.Errorf("snapshot %v has nil tree", sn.ID().Str())
@@ -78,33 +113,50 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti
 		return false, err
 	}
 
-	selectByName := func(nodepath string) bool {
-		for _, reject := range rejectByNameFuncs {
-			if reject(nodepath) {
-				return false
-			}
-		}
-		return true
+	metadata, err := opts.Metadata.convert()
+
+	if err != nil {
+		return false, err
 	}
 
-	rewriter := walker.NewTreeRewriter(walker.RewriteOpts{
-		RewriteNode: func(node *restic.Node, path string) *restic.Node {
-			if selectByName(path) {
-				return node
+	var filter rewriteFilterFunc
+
+	if len(rejectByNameFuncs) > 0 {
+		selectByName := func(nodepath string) bool {
+			for _, reject := range rejectByNameFuncs {
+				if reject(nodepath) {
+					return false
+				}
 			}
-			Verbosef(fmt.Sprintf("excluding %s\n", path))
-			return nil
-		},
-		DisableNodeCache: true,
-	})
+			return true
+		}
+
+		rewriter := walker.NewTreeRewriter(walker.RewriteOpts{
+			RewriteNode: func(node *restic.Node, path string) *restic.Node {
+				if selectByName(path) {
+					return node
+				}
+				Verbosef(fmt.Sprintf("excluding %s\n", path))
+				return nil
+			},
+			DisableNodeCache: true,
+		})
+
+		filter = func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
+			return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
+		}
+	} else {
+		filter = func(_ context.Context, sn *restic.Snapshot) (restic.ID, error) {
+			return *sn.Tree, nil
+		}
+	}
 
 	return filterAndReplaceSnapshot(ctx, repo, sn,
-		func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) {
-			return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree)
-		}, opts.DryRun, opts.Forget, "rewrite")
+		filter, opts.DryRun, opts.Forget, metadata, "rewrite")
 }
 
-func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot, filter func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error), dryRun bool, forget bool, addTag string) (bool, error) {
+func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *restic.Snapshot,
+	filter rewriteFilterFunc, dryRun bool, forget bool, newMetadata *snapshotMetadata, addTag string) (bool, error) {
 
 	wg, wgCtx := errgroup.WithContext(ctx)
 	repo.StartPackUploader(wgCtx, wg)
@@ -128,8 +180,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 		if dryRun {
 			Verbosef("would delete empty snapshot\n")
 		} else {
-			h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
-			if err = repo.Backend().Remove(ctx, h); err != nil {
+			if err = repo.RemoveUnpacked(ctx, restic.SnapshotFile, *sn.ID()); err != nil {
 				return false, err
 			}
 			debug.Log("removed empty snapshot %v", sn.ID())
@@ -138,7 +189,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 		return true, nil
 	}
 
-	if filteredTree == *sn.Tree {
+	if filteredTree == *sn.Tree && newMetadata == nil {
 		debug.Log("Snapshot %v not modified", sn)
 		return false, nil
 	}
@@ -151,6 +202,14 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 			Verbosef("would remove old snapshot\n")
 		}
 
+		if newMetadata != nil && newMetadata.Time != nil {
+			Verbosef("would set time to %s\n", newMetadata.Time)
+		}
+
+		if newMetadata != nil && newMetadata.Hostname != "" {
+			Verbosef("would set hostname to %s\n", newMetadata.Hostname)
+		}
+
 		return true, nil
 	}
 
@@ -162,6 +221,16 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 		sn.AddTags([]string{addTag})
 	}
 
+	if newMetadata != nil && newMetadata.Time != nil {
+		Verbosef("setting time to %s\n", *newMetadata.Time)
+		sn.Time = *newMetadata.Time
+	}
+
+	if newMetadata != nil && newMetadata.Hostname != "" {
+		Verbosef("setting host to %s\n", newMetadata.Hostname)
+		sn.Hostname = newMetadata.Hostname
+	}
+
 	// Save the new snapshot.
 	id, err := restic.SaveSnapshot(ctx, repo, sn)
 	if err != nil {
@@ -170,8 +239,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 	Verbosef("saved new snapshot %v\n", id.Str())
 
 	if forget {
-		h := restic.Handle{Type: restic.SnapshotFile, Name: sn.ID().String()}
-		if err = repo.Backend().Remove(ctx, h); err != nil {
+		if err = repo.RemoveUnpacked(ctx, restic.SnapshotFile, *sn.ID()); err != nil {
 			return false, err
 		}
 		debug.Log("removed old snapshot %v", sn.ID())
@@ -181,33 +249,28 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r
 }
 
 func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, args []string) error {
-	if opts.excludePatternOptions.Empty() {
-		return errors.Fatal("Nothing to do: no excludes provided")
+	if opts.excludePatternOptions.Empty() && opts.Metadata.empty() {
+		return errors.Fatal("Nothing to do: no excludes provided and no new metadata provided")
 	}
 
-	repo, err := OpenRepository(ctx, gopts)
+	var (
+		repo   *repository.Repository
+		unlock func()
+		err    error
+	)
+
+	if opts.Forget {
+		Verbosef("create exclusive lock for repository\n")
+		ctx, repo, unlock, err = openWithExclusiveLock(ctx, gopts, opts.DryRun)
+	} else {
+		ctx, repo, unlock, err = openWithAppendLock(ctx, gopts, opts.DryRun)
+	}
 	if err != nil {
 		return err
 	}
+	defer unlock()
 
-	if !opts.DryRun {
-		var lock *restic.Lock
-		var err error
-		if opts.Forget {
-			Verbosef("create exclusive lock for repository\n")
-			lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
-		} else {
-			lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		}
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	} else {
-		repo.SetDryRun()
-	}
-
-	snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
+	snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
 	if err != nil {
 		return err
 	}
@@ -219,7 +282,7 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a
 
 	changedCount := 0
 	for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) {
-		Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
+		Verbosef("\n%v\n", sn)
 		changed, err := rewriteSnapshot(ctx, repo, sn, opts)
 		if err != nil {
 			return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err)
@@ -228,6 +291,9 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a
 			changedCount++
 		}
 	}
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 
 	Verbosef("\n")
 	if changedCount == 0 {

cmd/restic/cmd_rewrite_integration_test.go

@@ -9,12 +9,13 @@ import (
 	rtest "github.com/restic/restic/internal/test"
 )
 
-func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool) {
+func testRunRewriteExclude(t testing.TB, gopts GlobalOptions, excludes []string, forget bool, metadata snapshotMetadataArgs) {
 	opts := RewriteOptions{
 		excludePatternOptions: excludePatternOptions{
 			Excludes: excludes,
 		},
-		Forget: forget,
+		Forget:   forget,
+		Metadata: metadata,
 	}
 
 	rtest.OK(t, runRewrite(context.TODO(), opts, gopts, nil))
@@ -38,7 +39,7 @@ func TestRewrite(t *testing.T) {
 	createBasicRewriteRepo(t, env)
 
 	// exclude some data
-	testRunRewriteExclude(t, env.gopts, []string{"3"}, false)
+	testRunRewriteExclude(t, env.gopts, []string{"3"}, false, snapshotMetadataArgs{Hostname: "", Time: ""})
 	snapshotIDs := testRunList(t, "snapshots", env.gopts)
 	rtest.Assert(t, len(snapshotIDs) == 2, "expected two snapshots, got %v", snapshotIDs)
 	testRunCheck(t, env.gopts)
@@ -50,7 +51,7 @@ func TestRewriteUnchanged(t *testing.T) {
 	snapshotID := createBasicRewriteRepo(t, env)
 
 	// use an exclude that will not exclude anything
-	testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false)
+	testRunRewriteExclude(t, env.gopts, []string{"3dflkhjgdflhkjetrlkhjgfdlhkj"}, false, snapshotMetadataArgs{Hostname: "", Time: ""})
 	newSnapshotIDs := testRunList(t, "snapshots", env.gopts)
 	rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs)
 	rtest.Assert(t, snapshotID == newSnapshotIDs[0], "snapshot id changed unexpectedly")
@@ -63,11 +64,47 @@ func TestRewriteReplace(t *testing.T) {
 	snapshotID := createBasicRewriteRepo(t, env)
 
 	// exclude some data
-	testRunRewriteExclude(t, env.gopts, []string{"3"}, true)
-	newSnapshotIDs := testRunList(t, "snapshots", env.gopts)
-	rtest.Assert(t, len(newSnapshotIDs) == 1, "expected one snapshot, got %v", newSnapshotIDs)
+	testRunRewriteExclude(t, env.gopts, []string{"3"}, true, snapshotMetadataArgs{Hostname: "", Time: ""})
+	newSnapshotIDs := testListSnapshots(t, env.gopts, 1)
 	rtest.Assert(t, snapshotID != newSnapshotIDs[0], "snapshot id should have changed")
 	// check forbids unused blobs, thus remove them first
 	testRunPrune(t, env.gopts, PruneOptions{MaxUnused: "0"})
 	testRunCheck(t, env.gopts)
 }
+
+func testRewriteMetadata(t *testing.T, metadata snapshotMetadataArgs) {
+	env, cleanup := withTestEnvironment(t)
+	defer cleanup()
+	createBasicRewriteRepo(t, env)
+	testRunRewriteExclude(t, env.gopts, []string{}, true, metadata)
+
+	ctx, repo, unlock, err := openWithReadLock(context.TODO(), env.gopts, false)
+	rtest.OK(t, err)
+	defer unlock()
+
+	snapshots, err := restic.TestLoadAllSnapshots(ctx, repo, nil)
+	rtest.OK(t, err)
+	rtest.Assert(t, len(snapshots) == 1, "expected one snapshot, got %v", len(snapshots))
+	newSnapshot := snapshots[0]
+
+	if metadata.Time != "" {
+		rtest.Assert(t, newSnapshot.Time.Format(TimeFormat) == metadata.Time, "New snapshot should have time %s", metadata.Time)
+	}
+
+	if metadata.Hostname != "" {
+		rtest.Assert(t, newSnapshot.Hostname == metadata.Hostname, "New snapshot should have host %s", metadata.Hostname)
+	}
+}
+
+func TestRewriteMetadata(t *testing.T) {
+	newHost := "new host"
+	newTime := "1999-01-01 11:11:11"
+
+	for _, metadata := range []snapshotMetadataArgs{
+		{Hostname: "", Time: newTime},
+		{Hostname: newHost, Time: ""},
+		{Hostname: newHost, Time: newTime},
+	} {
+		testRewriteMetadata(t, metadata)
+	}
+}

cmd/restic/cmd_snapshots.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/restic/restic/internal/restic"
+	"github.com/restic/restic/internal/ui"
 	"github.com/restic/restic/internal/ui/table"
 	"github.com/spf13/cobra"
 )
@@ -58,24 +59,19 @@ func init() {
 }
 
 func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions, args []string) error {
-	repo, err := OpenRepository(ctx, gopts)
+	ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
 	if err != nil {
 		return err
 	}
-
-	if !gopts.NoLock {
-		var lock *restic.Lock
-		lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
-		defer unlockRepo(lock)
-		if err != nil {
-			return err
-		}
-	}
+	defer unlock()
 
 	var snapshots restic.Snapshots
-	for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) {
+	for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
 		snapshots = append(snapshots, sn)
 	}
+	if ctx.Err() != nil {
+		return ctx.Err()
+	}
 	snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy)
 	if err != nil {
 		return err
@@ -163,6 +159,11 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
 			keepReasons[*id] = reasons[i]
 		}
 	}
+	// check if any snapshot contains a summary
+	hasSize := false
+	for _, sn := range list {
+		hasSize = hasSize || (sn.Summary != nil)
+	}
 
 	// always sort the snapshots so that the newer ones are listed last
 	sort.SliceStable(list, func(i, j int) bool {
@@ -198,6 +199,9 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
 			tab.AddColumn("Reasons", `{{ join .Reasons "\n" }}`)
 		}
 		tab.AddColumn("Paths", `{{ join .Paths "\n" }}`)
+		if hasSize {
+			tab.AddColumn("Size", `{{ .Size }}`)
+		}
 	}
 
 	type snapshot struct {
@@ -207,6 +211,7 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
 		Tags      []string
 		Reasons   []string
 		Paths     []string
+		Size      string
 	}
 
 	var multiline bool
@@ -228,6 +233,10 @@ func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.Ke
 			multiline = true
 		}
 
+		if sn.Summary != nil {
+			data.Size = ui.FormatBytes(sn.Summary.TotalBytesProcessed)
+		}
+
 		tab.AddRow(data)
 	}
 
@@ -290,7 +299,7 @@ func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string) error {
 	return nil
 }
 
-// Snapshot helps to print Snaphots as JSON with their ID included.
+// Snapshot helps to print Snapshots as JSON with their ID included.
 type Snapshot struct {
 	*restic.Snapshot