From 62f99a3b2f847e46bfe03102553c3bbaf203a96d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 7 Jan 2024 10:16:30 +0100 Subject: [PATCH 001/158] Add documentation for the verify-release-binaries.sh script --- doc/developer_information.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/doc/developer_information.rst b/doc/developer_information.rst index 9de517901..e7b60a03b 100644 --- a/doc/developer_information.rst +++ b/doc/developer_information.rst @@ -113,6 +113,26 @@ The following steps are necessary to build the binaries: restic/builder \ go run helpers/build-release-binaries/main.go --version 0.14.0 --verbose +Verifying the Official Binaries +******************************* + +To verify the official binaries, you can either build them yourself using the above +instructions or use the ``helpers/verify-release-binaries.sh`` script from the restic +repository. Run it as ``helpers/verify-release-binaries.sh restic_version go_version``. +The specified go compiler version must match the one used to build the official +binaries. For example, for restic 0.16.2 the command would be +``helpers/verify-release-binaries.sh 0.16.2 1.21.3``. + +The script requires bash, curl, docker, git, gpg, shasum and tar. + +The script first downloads all release binaries, checks the SHASUM256 file and its +signature. Afterwards it checks that the tarball matches the restic git repository +contents, before first reproducing the builder docker container and finally the +restic binaries. As final step, the restic binary in both the docker hub images +and the GitHub container registry is verified. If any step fails, then the script +will issue a warning. + + Prepare a New Release ********************* From 5cd000f4b06bbb3184ec26590584ecb8feb5a750 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 23 Dec 2023 13:40:04 +0100 Subject: [PATCH 002/158] CI: update golangci-lint Necessary to properly support Go 1.21. --- .github/workflows/tests.yml | 2 +- cmd/restic/integration_helpers_test.go | 3 +-- internal/fuse/snapshots_dir.go | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index e81aefdb5..291fab0f0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -255,7 +255,7 @@ jobs: uses: golangci/golangci-lint-action@v3 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.52.2 + version: v1.55.2 args: --verbose --timeout 5m # only run golangci-lint for pull requests, otherwise ALL hints get diff --git a/cmd/restic/integration_helpers_test.go b/cmd/restic/integration_helpers_test.go index 2afdbb938..d97589e80 100644 --- a/cmd/restic/integration_helpers_test.go +++ b/cmd/restic/integration_helpers_test.go @@ -123,9 +123,8 @@ func directoriesContentsDiff(dir1, dir2 string) string { fmt.Fprintf(&out, "+%v\n", b.path) b = nil continue - } else { - fmt.Fprintf(&out, "%%%v\n", a.path) } + fmt.Fprintf(&out, "%%%v\n", a.path) } a, b = nil, nil diff --git a/internal/fuse/snapshots_dir.go b/internal/fuse/snapshots_dir.go index 61df3ad08..7369ea17a 100644 --- a/internal/fuse/snapshots_dir.go +++ b/internal/fuse/snapshots_dir.go @@ -110,9 +110,8 @@ func (d *SnapshotsDir) Lookup(ctx context.Context, name string) (fs.Node, error) return newSnapshotLink(d.root, inode, entry.linkTarget, entry.snapshot) } else if entry.snapshot != nil { return newDirFromSnapshot(d.root, inode, entry.snapshot) - } else { - return NewSnapshotsDir(d.root, inode, d.inode, d.dirStruct, d.prefix+"/"+name), nil } + return NewSnapshotsDir(d.root, inode, d.inode, d.dirStruct, d.prefix+"/"+name), nil } return nil, syscall.ENOENT From 3fa6b2de4a2ea2b4660e0d5d94d995c5831c17ef Mon Sep 17 00:00:00 2001 From: Michael Kuhn Date: Mon, 20 Nov 2023 22:56:27 +0100 Subject: [PATCH 003/158] Fix repository not being printed when using repository file When using `RESTIC_REPOSITORY_FILE` in combination with `restic init`, the repository is missing in the output: ``` $ restic init created restic repository 3c872be20f at [...] ``` This is due to the code using `gopts.Repo`, which is empty in this case. --- cmd/restic/cmd_init.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/restic/cmd_init.go b/cmd/restic/cmd_init.go index b9dabdc2d..7154279e8 100644 --- a/cmd/restic/cmd_init.go +++ b/cmd/restic/cmd_init.go @@ -75,7 +75,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args [] return err } - repo, err := ReadRepo(gopts) + gopts.Repo, err = ReadRepo(gopts) if err != nil { return err } @@ -87,7 +87,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args [] return err } - be, err := create(ctx, repo, gopts, gopts.extended) + be, err := create(ctx, gopts.Repo, gopts, gopts.extended) if err != nil { return errors.Fatalf("create repository at %s failed: %v\n", location.StripPassword(gopts.backends, gopts.Repo), err) } From 64b7aed36299c1409e837d19a3fcbfd49b071e74 Mon Sep 17 00:00:00 2001 From: Quang-Linh LE Date: Fri, 24 Nov 2023 15:29:39 +0100 Subject: [PATCH 004/158] docs: Mention progress for restore command. This is available after https://github.com/restic/restic/pull/3991 --- doc/manual_rest.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index 71f5e192b..d1c64ba6e 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -148,11 +148,11 @@ command: -v, --verbose be verbose (specify multiple times or a level using --verbose=n, max level/times is 2) Subcommands that support showing progress information such as ``backup``, -``check`` and ``prune`` will do so unless the quiet flag ``-q`` or -``--quiet`` is set. When running from a non-interactive console progress -reporting is disabled by default to not fill your logs. For interactive -and non-interactive consoles the environment variable ``RESTIC_PROGRESS_FPS`` -can be used to control the frequency of progress reporting. Use for example +``restore``, ``check`` and ``prune`` will do so unless the quiet flag ``-q`` +or ``--quiet`` is set. When running from a non-interactive console progress +reporting is disabled by default to not fill your logs. For interactive and +non-interactive consoles the environment variable ``RESTIC_PROGRESS_FPS`` can +be used to control the frequency of progress reporting. Use for example ``0.016666`` to only update the progress once per minute. Additionally, on Unix systems if ``restic`` receives a SIGUSR1 signal the From 76bd975e03ac1ac20adeed8fdc7a24cf513965d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joda=20St=C3=B6=C3=9Fer?= Date: Mon, 18 Dec 2023 23:04:46 +0100 Subject: [PATCH 005/158] docs(scripting): correct stats output comment to be about the correct command not about the snapshots command --- doc/075_scripting.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index 71ecd4c2b..fe41ac870 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -556,7 +556,7 @@ The snapshots command returns a single JSON object, an array with objects of the stats ----- -The snapshots command returns a single JSON object. +The stats command returns a single JSON object. +------------------------------+-----------------------------------------------------+ | ``total_size`` | Repository size in bytes | From f44fd73230a944133e7945df9a534034548752ec Mon Sep 17 00:00:00 2001 From: Giuseppe D'Andrea Date: Tue, 5 Dec 2023 16:10:39 +0100 Subject: [PATCH 006/158] docs: fix typo in working with repos When using the `copy` command, `--from-password-file` and `--from-password-command` flags are used to specify the password of the source repository, not of the destination repository. --- doc/045_working_with_repos.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index 3f6b1090e..77c7a15b5 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -120,7 +120,7 @@ be skipped by later copy runs. The source repository is specified with ``--from-repo`` or can be read from a file specified via ``--from-repository-file``. Both of these options can also be set as environment variables ``$RESTIC_FROM_REPOSITORY`` or -``$RESTIC_FROM_REPOSITORY_FILE``, respectively. For the destination repository +``$RESTIC_FROM_REPOSITORY_FILE``, respectively. For the source repository the password can be read from a file ``--from-password-file`` or from a command ``--from-password-command``. Alternatively the environment variables ``$RESTIC_FROM_PASSWORD_COMMAND`` and From 220a28582e705559df27e09e2614c4e0724dbd1b Mon Sep 17 00:00:00 2001 From: mmattel Date: Tue, 14 Nov 2023 12:39:17 +0100 Subject: [PATCH 007/158] Add a table of contents (TOC) to the changelog template --- changelog/CHANGELOG.tmpl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/changelog/CHANGELOG.tmpl b/changelog/CHANGELOG.tmpl index 712e7cc54..2c7ac353b 100644 --- a/changelog/CHANGELOG.tmpl +++ b/changelog/CHANGELOG.tmpl @@ -1,18 +1,20 @@ -{{- range $changes := . }}{{ with $changes -}} -Changelog for restic {{ .Version }} ({{ .Date }}) -======================================= +# Table of Contents +{{ range . -}} + * [Changelog for {{ .Version }}](#changelog-for-restic-{{ .Version | replace "." ""}}-{{ .Date | lower -}}) +{{ end -}} +{{- range $changes := . }}{{ with $changes -}} + +# Changelog for restic {{ .Version }} ({{ .Date }}) The following sections list the changes in restic {{ .Version }} relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary {{ range $entry := .Entries }}{{ with $entry }} * {{ .TypeShort }} #{{ .PrimaryID }}: {{ .Title }} {{- end }}{{ end }} -Details -------- +## Details {{ range $entry := .Entries }}{{ with $entry }} * {{ .Type }} #{{ .PrimaryID }}: {{ .Title }} {{ range $par := .Paragraphs }} From 7e61e117d64451bf02c22b078b3910b96bd0c5f3 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 24 Dec 2023 11:44:20 +0100 Subject: [PATCH 008/158] cleanup changelog whitespace --- changelog/CHANGELOG.tmpl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog/CHANGELOG.tmpl b/changelog/CHANGELOG.tmpl index 2c7ac353b..20f6254e7 100644 --- a/changelog/CHANGELOG.tmpl +++ b/changelog/CHANGELOG.tmpl @@ -3,7 +3,8 @@ {{ range . -}} * [Changelog for {{ .Version }}](#changelog-for-restic-{{ .Version | replace "." ""}}-{{ .Date | lower -}}) {{ end -}} -{{- range $changes := . }}{{ with $changes -}} + +{{- range $changes := . }}{{ with $changes }} # Changelog for restic {{ .Version }} ({{ .Date }}) The following sections list the changes in restic {{ .Version }} relevant to @@ -29,6 +30,5 @@ restic users. The changes are ordered by importance. {{ range $url := .OtherURLs }} {{ $url -}} {{ end }} -{{ end }}{{ end }} - +{{ end }}{{ end -}} {{ end }}{{ end -}} From ff0c9754435e796642060f2321e078473edbaadb Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 24 Dec 2023 11:45:26 +0100 Subject: [PATCH 009/158] regenerate changelog --- CHANGELOG.md | 323 +++++++++++++++++++-------------------------------- 1 file changed, 119 insertions(+), 204 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab07cc225..872b70a1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,17 +1,46 @@ -Changelog for restic 0.16.2 (2023-10-29) -======================================= +# Table of Contents +* [Changelog for 0.16.2](#changelog-for-restic-0162-2023-10-29) +* [Changelog for 0.16.1](#changelog-for-restic-0161-2023-10-24) +* [Changelog for 0.16.0](#changelog-for-restic-0160-2023-07-31) +* [Changelog for 0.15.2](#changelog-for-restic-0152-2023-04-24) +* [Changelog for 0.15.1](#changelog-for-restic-0151-2023-01-30) +* [Changelog for 0.15.0](#changelog-for-restic-0150-2023-01-12) +* [Changelog for 0.14.0](#changelog-for-restic-0140-2022-08-25) +* [Changelog for 0.13.0](#changelog-for-restic-0130-2022-03-26) +* [Changelog for 0.12.1](#changelog-for-restic-0121-2021-08-03) +* [Changelog for 0.12.0](#changelog-for-restic-0120-2021-02-14) +* [Changelog for 0.11.0](#changelog-for-restic-0110-2020-11-05) +* [Changelog for 0.10.0](#changelog-for-restic-0100-2020-09-19) +* [Changelog for 0.9.6](#changelog-for-restic-096-2019-11-22) +* [Changelog for 0.9.5](#changelog-for-restic-095-2019-04-23) +* [Changelog for 0.9.4](#changelog-for-restic-094-2019-01-06) +* [Changelog for 0.9.3](#changelog-for-restic-093-2018-10-13) +* [Changelog for 0.9.2](#changelog-for-restic-092-2018-08-06) +* [Changelog for 0.9.1](#changelog-for-restic-091-2018-06-10) +* [Changelog for 0.9.0](#changelog-for-restic-090-2018-05-21) +* [Changelog for 0.8.3](#changelog-for-restic-083-2018-02-26) +* [Changelog for 0.8.2](#changelog-for-restic-082-2018-02-17) +* [Changelog for 0.8.1](#changelog-for-restic-081-2017-12-27) +* [Changelog for 0.8.0](#changelog-for-restic-080-2017-11-26) +* [Changelog for 0.7.3](#changelog-for-restic-073-2017-09-20) +* [Changelog for 0.7.2](#changelog-for-restic-072-2017-09-13) +* [Changelog for 0.7.1](#changelog-for-restic-071-2017-07-22) +* [Changelog for 0.7.0](#changelog-for-restic-070-2017-07-01) +* [Changelog for 0.6.1](#changelog-for-restic-061-2017-06-01) +* [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29) + + +# Changelog for restic 0.16.2 (2023-10-29) The following sections list the changes in restic 0.16.2 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #4540: Restore ARMv5 support for ARM binaries * Fix #4545: Repair documentation build on Read the Docs -Details -------- +## Details * Bugfix #4540: Restore ARMv5 support for ARM binaries @@ -31,14 +60,11 @@ Details https://github.com/restic/restic/pull/4545 -Changelog for restic 0.16.1 (2023-10-24) -======================================= - +# Changelog for restic 0.16.1 (2023-10-24) The following sections list the changes in restic 0.16.1 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #4513: Make `key list` command honor `--no-lock` * Fix #4516: Do not try to load password on command line autocomplete @@ -50,8 +76,7 @@ Summary * Enh #4511: Include inode numbers in JSON output for `find` and `ls` commands * Enh #4519: Add config option to set SFTP command arguments -Details -------- +## Details * Bugfix #4513: Make `key list` command honor `--no-lock` @@ -152,14 +177,11 @@ Details https://github.com/restic/restic/pull/4519 -Changelog for restic 0.16.0 (2023-07-31) -======================================= - +# Changelog for restic 0.16.0 (2023-07-31) The following sections list the changes in restic 0.16.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #2565: Support "unlimited" in `forget --keep-*` options * Fix #3311: Support non-UTF8 paths as symlink target @@ -191,8 +213,7 @@ Summary * Enh #4226: Allow specifying region of new buckets in the `gs` backend * Enh #4375: Add support for extended attributes on symlinks -Details -------- +## Details * Bugfix #2565: Support "unlimited" in `forget --keep-*` options @@ -499,14 +520,11 @@ Details https://github.com/restic/restic/pull/4379 -Changelog for restic 0.15.2 (2023-04-24) -======================================= - +# Changelog for restic 0.15.2 (2023-04-24) The following sections list the changes in restic 0.15.2 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Sec #4275: Update golang.org/x/net to address CVE-2022-41723 * Fix #2260: Sanitize filenames printed by `backup` during processing @@ -516,8 +534,7 @@ Summary * Enh #4180: Add release binaries for riscv64 architecture on Linux * Enh #4219: Upgrade Minio to version 7.0.49 -Details -------- +## Details * Security #4275: Update golang.org/x/net to address CVE-2022-41723 @@ -588,14 +605,11 @@ Details https://github.com/restic/restic/pull/4219 -Changelog for restic 0.15.1 (2023-01-30) -======================================= - +# Changelog for restic 0.15.1 (2023-01-30) The following sections list the changes in restic 0.15.1 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #3750: Remove `b2_download_file_by_name: 404` warning from B2 backend * Fix #4147: Make `prune --quiet` not print progress bar @@ -603,8 +617,7 @@ Summary * Fix #4167: Add missing ETA in `backup` progress bar * Enh #4143: Ignore empty lock files -Details -------- +## Details * Bugfix #3750: Remove `b2_download_file_by_name: 404` warning from B2 backend @@ -663,14 +676,11 @@ Details https://github.com/restic/restic/pull/4152 -Changelog for restic 0.15.0 (2023-01-12) -======================================= - +# Changelog for restic 0.15.0 (2023-01-12) The following sections list the changes in restic 0.15.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #2015: Make `mount` return exit code 0 after receiving Ctrl-C / SIGINT * Fix #2578: Make `restore` replace existing symlinks @@ -712,8 +722,7 @@ Summary * Enh #3943: Ignore additional/unknown files in repository * Enh #3955: Improve `backup` performance for small files -Details -------- +## Details * Bugfix #2015: Make `mount` return exit code 0 after receiving Ctrl-C / SIGINT @@ -1144,14 +1153,11 @@ Details https://github.com/restic/restic/pull/3955 -Changelog for restic 0.14.0 (2022-08-25) -======================================= - +# Changelog for restic 0.14.0 (2022-08-25) The following sections list the changes in restic 0.14.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #2248: Support `self-update` on Windows * Fix #3428: List snapshots in backend at most once to resolve snapshot IDs @@ -1187,8 +1193,7 @@ Summary * Enh #3819: Validate include/exclude patterns before restoring * Enh #3837: Improve SFTP repository initialization over slow links -Details -------- +## Details * Bugfix #2248: Support `self-update` on Windows @@ -1594,14 +1599,11 @@ Details https://github.com/restic/restic/pull/3840 -Changelog for restic 0.13.0 (2022-03-26) -======================================= - +# Changelog for restic 0.13.0 (2022-03-26) The following sections list the changes in restic 0.13.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1106: Never lock repository for `list locks` * Fix #2345: Make cache crash-resistant and usable by multiple concurrent processes @@ -1638,8 +1640,7 @@ Summary * Enh #3542: Add file mode in symbolic notation to `ls --json` * Enh #3593: Improve `copy` performance by parallelizing IO -Details -------- +## Details * Bugfix #1106: Never lock repository for `list locks` @@ -1999,14 +2000,11 @@ Details https://github.com/restic/restic/pull/3593 -Changelog for restic 0.12.1 (2021-08-03) -======================================= - +# Changelog for restic 0.12.1 (2021-08-03) The following sections list the changes in restic 0.12.1 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #2742: Improve error handling for rclone and REST backend over HTTP2 * Fix #3111: Fix terminal output redirection for PowerShell @@ -2031,8 +2029,7 @@ Summary * Enh #3427: `find --pack` fallback to index if data file is missing * Enh #3456: Support filtering and specifying untagged snapshots -Details -------- +## Details * Bugfix #2742: Improve error handling for rclone and REST backend over HTTP2 @@ -2245,14 +2242,11 @@ Details https://github.com/restic/restic/pull/3457 -Changelog for restic 0.12.0 (2021-02-14) -======================================= - +# Changelog for restic 0.12.0 (2021-02-14) The following sections list the changes in restic 0.12.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1681: Make `mount` not create missing mount point directory * Fix #1800: Ignore `no data available` filesystem error during backup @@ -2290,8 +2284,7 @@ Summary * Enh #3250: Add several more error checks * Enh #3254: Enable HTTP/2 for backend connections -Details -------- +## Details * Bugfix #1681: Make `mount` not create missing mount point directory @@ -2742,14 +2735,11 @@ Details https://github.com/restic/restic/pull/3254 -Changelog for restic 0.11.0 (2020-11-05) -======================================= - +# Changelog for restic 0.11.0 (2020-11-05) The following sections list the changes in restic 0.11.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1212: Restore timestamps and permissions on intermediate directories * Fix #1756: Mark repository files as read-only when using the local backend @@ -2767,8 +2757,7 @@ Summary * Enh #2969: Optimize check for unchanged files during backup * Enh #2978: Warn if parent snapshot cannot be loaded during backup -Details -------- +## Details * Bugfix #1212: Restore timestamps and permissions on intermediate directories @@ -2917,14 +2906,11 @@ Details https://github.com/restic/restic/pull/2978 -Changelog for restic 0.10.0 (2020-09-19) -======================================= - +# Changelog for restic 0.10.0 (2020-09-19) The following sections list the changes in restic 0.10.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1863: Report correct number of directories processed by backup * Fix #2254: Fix tar issues when dumping `/` @@ -2971,8 +2957,7 @@ Summary * Enh #2840: Speed-up file deletion in forget, prune and rebuild-index * Enh #2858: Support filtering snapshots by tag and path in the stats command -Details -------- +## Details * Bugfix #1863: Report correct number of directories processed by backup @@ -3381,14 +3366,11 @@ Details https://forum.restic.net/t/stats-for-a-host-and-filtered-snapshots/3020 -Changelog for restic 0.9.6 (2019-11-22) -======================================= - +# Changelog for restic 0.9.6 (2019-11-22) The following sections list the changes in restic 0.9.6 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #2063: Allow absolute path for filename when backing up from stdin * Fix #2174: Save files with invalid timestamps @@ -3400,8 +3382,7 @@ Summary * Enh #2330: Make `--group-by` accept both singular and plural * Enh #2350: Add option to configure S3 region -Details -------- +## Details * Bugfix #2063: Allow absolute path for filename when backing up from stdin @@ -3484,14 +3465,11 @@ Details https://github.com/restic/restic/pull/2350 -Changelog for restic 0.9.5 (2019-04-23) -======================================= - +# Changelog for restic 0.9.5 (2019-04-23) The following sections list the changes in restic 0.9.5 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #2135: Return error when no bytes could be read from stdin * Fix #2181: Don't cancel timeout after 30 seconds for self-update @@ -3507,8 +3485,7 @@ Summary * Enh #2205: Add --ignore-inode option to backup cmd * Enh #2220: Add config option to set S3 storage class -Details -------- +## Details * Bugfix #2135: Return error when no bytes could be read from stdin @@ -3626,14 +3603,11 @@ Details https://github.com/restic/restic/pull/2220 -Changelog for restic 0.9.4 (2019-01-06) -======================================= - +# Changelog for restic 0.9.4 (2019-01-06) The following sections list the changes in restic 0.9.4 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1989: Google Cloud Storage: Respect bandwidth limit * Fix #2040: Add host name filter shorthand flag for `stats` command @@ -3647,8 +3621,7 @@ Summary * Enh #2094: Run command to get password * Enh #2097: Add key hinting -Details -------- +## Details * Bugfix #1989: Google Cloud Storage: Respect bandwidth limit @@ -3754,14 +3727,11 @@ Details https://github.com/restic/restic/issues/2097 -Changelog for restic 0.9.3 (2018-10-13) -======================================= - +# Changelog for restic 0.9.3 (2018-10-13) The following sections list the changes in restic 0.9.3 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1935: Remove truncated files from cache * Fix #1978: Do not return an error when the scanner is slower than backup @@ -3778,8 +3748,7 @@ Summary * Enh #1967: Use `--host` everywhere * Enh #2028: Display size of cache directories -Details -------- +## Details * Bugfix #1935: Remove truncated files from cache @@ -3925,14 +3894,11 @@ Details https://github.com/restic/restic/pull/2033 -Changelog for restic 0.9.2 (2018-08-06) -======================================= - +# Changelog for restic 0.9.2 (2018-08-06) The following sections list the changes in restic 0.9.2 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1854: Allow saving files/dirs on different fs with `--one-file-system` * Fix #1861: Fix case-insensitive search with restic find @@ -3946,8 +3912,7 @@ Summary * Enh #1901: Update the Backblaze B2 library * Enh #1906: Add support for B2 application keys -Details -------- +## Details * Bugfix #1854: Allow saving files/dirs on different fs with `--one-file-system` @@ -4047,14 +4012,11 @@ Details https://github.com/restic/restic/pull/1914 -Changelog for restic 0.9.1 (2018-06-10) -======================================= - +# Changelog for restic 0.9.1 (2018-06-10) The following sections list the changes in restic 0.9.1 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1801: Add limiting bandwidth to the rclone backend * Fix #1822: Allow uploading large files to MS Azure @@ -4062,8 +4024,7 @@ Summary * Fix #1833: Fix caching files on error * Fix #1834: Resolve deadlock -Details -------- +## Details * Bugfix #1801: Add limiting bandwidth to the rclone backend @@ -4114,14 +4075,11 @@ Details https://github.com/restic/restic/pull/1835 -Changelog for restic 0.9.0 (2018-05-21) -======================================= - +# Changelog for restic 0.9.0 (2018-05-21) The following sections list the changes in restic 0.9.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1608: Respect time stamp for new backup when reading from stdin * Fix #1652: Ignore/remove invalid lock files @@ -4143,8 +4101,7 @@ Summary * Enh #1758: Allow saving OneDrive folders in Windows * Enh #1782: Use default AWS credentials chain for S3 backend -Details -------- +## Details * Bugfix #1608: Respect time stamp for new backup when reading from stdin @@ -4366,14 +4323,11 @@ Details https://github.com/restic/restic/pull/1782 -Changelog for restic 0.8.3 (2018-02-26) -======================================= - +# Changelog for restic 0.8.3 (2018-02-26) The following sections list the changes in restic 0.8.3 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1633: Fixed unexpected 'pack file cannot be listed' error * Fix #1638: Handle errors listing files in the backend @@ -4383,8 +4337,7 @@ Summary * Enh #1623: Don't check for presence of files in the backend before writing * Enh #1634: Upgrade B2 client library, reduce HTTP requests -Details -------- +## Details * Bugfix #1633: Fixed unexpected 'pack file cannot be listed' error @@ -4454,14 +4407,11 @@ Details https://github.com/restic/restic/pull/1634 -Changelog for restic 0.8.2 (2018-02-17) -======================================= - +# Changelog for restic 0.8.2 (2018-02-17) The following sections list the changes in restic 0.8.2 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1506: Limit bandwith at the http.RoundTripper for HTTP based backends * Fix #1512: Restore directory permissions as the last step @@ -4481,8 +4431,7 @@ Summary * Enh #1579: Retry Backend.List() in case of errors * Enh #1584: Limit index file size -Details -------- +## Details * Bugfix #1506: Limit bandwith at the http.RoundTripper for HTTP based backends @@ -4627,14 +4576,11 @@ Details https://github.com/restic/restic/pull/1584 -Changelog for restic 0.8.1 (2017-12-27) -======================================= - +# Changelog for restic 0.8.1 (2017-12-27) The following sections list the changes in restic 0.8.1 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1454: Correct cache dir location for Windows and Darwin * Fix #1457: Improve s3 backend with DigitalOcean Spaces @@ -4644,8 +4590,7 @@ Summary * Enh #1436: Add code to detect old cache directories * Enh #1439: Improve cancellation logic -Details -------- +## Details * Bugfix #1454: Correct cache dir location for Windows and Darwin @@ -4703,14 +4648,11 @@ Details https://github.com/restic/restic/pull/1439 -Changelog for restic 0.8.0 (2017-11-26) -======================================= - +# Changelog for restic 0.8.0 (2017-11-26) The following sections list the changes in restic 0.8.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Sec #1445: Prevent writing outside the target directory during restore * Fix #1256: Re-enable workaround for S3 backend @@ -4732,8 +4674,7 @@ Summary * Enh #1353: Retry failed backend requests * Enh #1367: Allow comments in files read from via `--file-from` -Details -------- +## Details * Security #1445: Prevent writing outside the target directory during restore @@ -4911,19 +4852,15 @@ Details https://github.com/restic/restic/pull/1368 -Changelog for restic 0.7.3 (2017-09-20) -======================================= - +# Changelog for restic 0.7.3 (2017-09-20) The following sections list the changes in restic 0.7.3 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1246: List all files stored in Google Cloud Storage -Details -------- +## Details * Bugfix #1246: List all files stored in Google Cloud Storage @@ -4935,14 +4872,11 @@ Details https://github.com/restic/restic/pull/1247 -Changelog for restic 0.7.2 (2017-09-13) -======================================= - +# Changelog for restic 0.7.2 (2017-09-13) The following sections list the changes in restic 0.7.2 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1164: Make the `key remove` command behave as documented * Fix #1167: Do not create a local repo unless `init` is used @@ -4962,8 +4896,7 @@ Summary * Enh #1205: Allow specifying time/date for a backup with `--time` * Enh #1218: Add `--compact` to `snapshots` command -Details -------- +## Details * Bugfix #1164: Make the `key remove` command behave as documented @@ -5071,14 +5004,11 @@ Details https://github.com/restic/restic/pull/1223 -Changelog for restic 0.7.1 (2017-07-22) -======================================= - +# Changelog for restic 0.7.1 (2017-07-22) The following sections list the changes in restic 0.7.1 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #1115: Fix `prune`, only include existing files in indexes * Enh #1055: Create subdirs below `data/` for local/sftp backends @@ -5088,8 +5018,7 @@ Summary * Enh #1081: Clarify semantic for `--tag` for the `forget` command * Enh #1082: Print stats on SIGINFO on Darwin and FreeBSD (ctrl+t) -Details -------- +## Details * Bugfix #1115: Fix `prune`, only include existing files in indexes @@ -5143,14 +5072,11 @@ Details https://github.com/restic/restic/pull/1082 -Changelog for restic 0.7.0 (2017-07-01) -======================================= - +# Changelog for restic 0.7.0 (2017-07-01) The following sections list the changes in restic 0.7.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Fix #965: Switch to `default` repo layout for the s3 backend * Fix #1013: Switch back to using the high-level minio-go API for s3 @@ -5162,8 +5088,7 @@ Summary * Enh #1021: Detect invalid backend name and print error * Enh #1029: Remove invalid pack files when `prune` is run -Details -------- +## Details * Bugfix #965: Switch to `default` repo layout for the s3 backend @@ -5234,21 +5159,17 @@ Details https://github.com/restic/restic/pull/1036 -Changelog for restic 0.6.1 (2017-06-01) -======================================= - +# Changelog for restic 0.6.1 (2017-06-01) The following sections list the changes in restic 0.6.1 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Enh #974: Remove regular status reports * Enh #981: Remove temporary path from binary in `build.go` * Enh #985: Allow multiple parallel idle HTTP connections -Details -------- +## Details * Enhancement #974: Remove regular status reports @@ -5275,21 +5196,17 @@ Details https://github.com/restic/restic/pull/986 -Changelog for restic 0.6.0 (2017-05-29) -======================================= - +# Changelog for restic 0.6.0 (2017-05-29) The following sections list the changes in restic 0.6.0 relevant to restic users. The changes are ordered by importance. -Summary -------- +## Summary * Enh #957: Make `forget` consistent * Enh #962: Improve memory and runtime for the s3 backend * Enh #966: Unify repository layout for all backends -Details -------- +## Details * Enhancement #957: Make `forget` consistent @@ -5325,5 +5242,3 @@ Details https://github.com/restic/restic/issues/965 https://github.com/restic/restic/pull/966 - - From bb0c9232988bd87874fff68eb373b68f4a126f9f Mon Sep 17 00:00:00 2001 From: Markus Zoppelt Date: Sun, 19 Nov 2023 11:03:42 +0100 Subject: [PATCH 010/158] docs: add pkgx install option PR in pkgx pantry: https://github.com/pkgxdev/pantry/pull/4098 restic pkg: https://pkgx.dev/pkgs/restic.net/restic/ --- doc/020_installation.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/020_installation.rst b/doc/020_installation.rst index 0fedc6425..0f1cd6c04 100644 --- a/doc/020_installation.rst +++ b/doc/020_installation.rst @@ -84,6 +84,12 @@ If you are using macOS, you can install restic using the $ brew install restic +On Linux and macOS, you can also install it using `pkgx `__: + +.. code-block:: console + + $ pkgx install restic + You may also install it using `MacPorts `__: .. code-block:: console From 948ab3ccaf603965d95cfb510c7b33497eeb4e11 Mon Sep 17 00:00:00 2001 From: Joram Berger Date: Mon, 18 Dec 2023 18:24:57 +0100 Subject: [PATCH 011/158] Add a note that the oldest snapshot may be kept additionally Documentation enhancement. --- doc/060_forget.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/060_forget.rst b/doc/060_forget.rst index 72c7ae97f..caeb6313a 100644 --- a/doc/060_forget.rst +++ b/doc/060_forget.rst @@ -207,10 +207,13 @@ The ``forget`` command accepts the following policy options: They also only count hours/days/weeks/etc which have one or more snapshots. A value of ``-1`` will be interpreted as "forever", i.e. "keep all". -.. note:: All duration related options (``--keep-{within,-*}``) ignore snapshots +.. note:: All duration related options (``--keep-{within-,}*``) ignore snapshots with a timestamp in the future (relative to when the ``forget`` command is run) and these snapshots will hence not be removed. +.. note:: If there are not enough snapshots to keep one for each duration related + ``--keep-{within-,}*`` option, the oldest snapshot is kept additionally. + .. note:: Specifying ``--keep-tag ''`` will match untagged snapshots only. When ``forget`` is run with a policy, restic first loads the list of all snapshots From d4b8abd3e254ebe0f3d97f3a94cea6a43239b59b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 19:32:38 +0100 Subject: [PATCH 012/158] fix deduplicated files on windows --- changelog/unreleased/issue-4574 | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 changelog/unreleased/issue-4574 diff --git a/changelog/unreleased/issue-4574 b/changelog/unreleased/issue-4574 new file mode 100644 index 000000000..3668ae6c3 --- /dev/null +++ b/changelog/unreleased/issue-4574 @@ -0,0 +1,11 @@ +Bugfix: support backup of deduplicated files on Windows again + +With the official release builds of restic 0.16.1 and 0.16.2, it was not +possible to back up files that were deduplicated by the corresponding Windows +Server feature. This also applies to restic versions built using Go +1.21.0 - 1.21.4. + +We have updated the used Go version to fix this. + +https://github.com/restic/restic/issues/4574 +https://github.com/restic/restic/pull/4621 From c95de547261f405715491297b90e607204b06be8 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 18:43:16 +0100 Subject: [PATCH 013/158] restic: cleanup node type determination os.ModeCharDevice is already included in os.ModeType --- internal/restic/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/restic/node.go b/internal/restic/node.go index edb49bfca..1b940b0d0 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -109,7 +109,7 @@ func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) { } func nodeTypeFromFileInfo(fi os.FileInfo) string { - switch fi.Mode() & (os.ModeType | os.ModeCharDevice) { + switch fi.Mode() & os.ModeType { case 0: return "file" case os.ModeDir: From e886c3f6b2b1c40418ca61bfef2c9d9aab3442cd Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 19:03:11 +0100 Subject: [PATCH 014/158] archiver: improve error message for irregular files Since Go 1.21, most reparse points are considered as irregular files. Depending on the underlying driver these can exhibit nearly arbitrary behavior. When encountering such a file, restic returned an indecipherable error message: `error: invalid node type ""`. Add the filepath to the error message and state that the file type is not supported. --- internal/archiver/archiver.go | 6 +++++- internal/restic/node.go | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index 98819d797..19b76c4f9 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -2,6 +2,7 @@ package archiver import ( "context" + "fmt" "os" "path" "runtime" @@ -183,7 +184,10 @@ func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo) } // overwrite name to match that within the snapshot node.Name = path.Base(snPath) - return node, errors.WithStack(err) + if err != nil { + return node, fmt.Errorf("nodeFromFileInfo %v: %w", filename, err) + } + return node, err } // loadSubtree tries to load the subtree referenced by node. In case of an error, nil is returned. diff --git a/internal/restic/node.go b/internal/restic/node.go index 1b940b0d0..7edc41ce8 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -124,6 +124,8 @@ func nodeTypeFromFileInfo(fi os.FileInfo) string { return "fifo" case os.ModeSocket: return "socket" + case os.ModeIrregular: + return "irregular" } return "" @@ -622,7 +624,7 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error { case "fifo": case "socket": default: - return errors.Errorf("invalid node type %q", node.Type) + return errors.Errorf("unsupported file type %q", node.Type) } return node.fillExtendedAttributes(path) From 1bab29c3365c7d17dcce8d6024f3cd43da947808 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 19:08:24 +0100 Subject: [PATCH 015/158] archiver: Add filepath to error message if it is not included yet --- internal/archiver/archiver.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index 19b76c4f9..d2ddbf00a 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -7,6 +7,7 @@ import ( "path" "runtime" "sort" + "strings" "time" "github.com/restic/restic/internal/debug" @@ -169,6 +170,11 @@ func (arch *Archiver) error(item string, err error) error { return err } + // not all errors include the filepath, thus add it if it is missing + if !strings.Contains(err.Error(), item) { + err = fmt.Errorf("%v: %w", item, err) + } + errf := arch.Error(item, err) if err != errf { debug.Log("item %v: error was filtered by handler, before: %q, after: %v", item, err, errf) From a9446c1184a60fe7780aca39237d95fd15aa7eb2 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 19:17:54 +0100 Subject: [PATCH 016/158] add changelog for irregular files on windows --- changelog/unreleased/issue-4560 | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 changelog/unreleased/issue-4560 diff --git a/changelog/unreleased/issue-4560 b/changelog/unreleased/issue-4560 new file mode 100644 index 000000000..c421f6e69 --- /dev/null +++ b/changelog/unreleased/issue-4560 @@ -0,0 +1,14 @@ +Bugfix: Improve errors for irregular files on Windows + +Since Go 1.21, most filesystem reparse points on Windows are considered to be +irregular files. This caused restic to show an `error: invalid node type ""` +error message for those files. + +We have improved the error message to include the file path for those files: +`error: nodeFromFileInfo path/to/file: unsupported file type "irregular"`. +As irregular files are not required to behave like regular files, it is not +possible to provide a generic way to back up those files. + +https://github.com/restic/restic/issues/4560 +https://github.com/restic/restic/pull/4620 +https://forum.restic.net/t/windows-backup-error-invalid-node-type/6875 From 772a907533f60fb943c8255c84752b4605fe8325 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 30 Dec 2023 21:40:41 +0100 Subject: [PATCH 017/158] repository: StreamPack delivers blobs at most once If an error occurred while streaming a pack file, this could result in passing some of the blobs multiple times to the callback function. This significantly complicates using StreamPack correctly and is unnecessary. Retries do not change the content of a blob and thus only deliver the same result over and over again. --- internal/repository/repository.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index d034911c5..0b50382b8 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -887,9 +887,9 @@ type BackendLoadFn func(ctx context.Context, h restic.Handle, length int, offset const maxUnusedRange = 4 * 1024 * 1024 // StreamPack loads the listed blobs from the specified pack file. The plaintext blob is passed to -// the handleBlobFn callback or an error if decryption failed or the blob hash does not match. In -// case of download errors handleBlobFn might be called multiple times for the same blob. If the -// callback returns an error, then StreamPack will abort and not retry it. +// the handleBlobFn callback or an error if decryption failed or the blob hash does not match. +// handleBlobFn is never called multiple times for the same blob. If the callback returns an error, +// then StreamPack will abort and not retry it. func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { if len(blobs) == 0 { // nothing to do @@ -951,7 +951,9 @@ func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, currentBlobEnd := dataStart var buf []byte var decode []byte - for _, entry := range blobs { + for len(blobs) > 0 { + entry := blobs[0] + skipBytes := int(entry.Offset - currentBlobEnd) if skipBytes < 0 { return errors.Errorf("overlapping blobs in pack %v", packID) @@ -1014,6 +1016,8 @@ func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, cancel() return backoff.Permanent(err) } + // ensure that each blob is only passed once to handleBlobFn + blobs = blobs[1:] } return nil }) From 7a3a884874d645bc74fa315c5b4d38e551a6066d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 30 Dec 2023 22:08:57 +0100 Subject: [PATCH 018/158] repository: test that StreamPack only delivers blobs once --- internal/repository/repository_test.go | 50 ++++++++++++++++++++------ 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index d362b0b5e..c4550d77d 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -5,6 +5,7 @@ import ( "context" "crypto/sha256" "encoding/json" + "errors" "fmt" "io" "math/rand" @@ -14,6 +15,7 @@ import ( "testing" "time" + "github.com/cenkalti/backoff/v4" "github.com/google/go-cmp/cmp" "github.com/klauspost/compress/zstd" "github.com/restic/restic/internal/backend/local" @@ -528,7 +530,9 @@ func testStreamPack(t *testing.T, version uint) { packfileBlobs, packfile := buildPackfileWithoutHeader(blobSizes, &key, compress) loadCalls := 0 - load := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + shortFirstLoad := false + + loadBytes := func(length int, offset int64) []byte { data := packfile if offset > int64(len(data)) { @@ -540,32 +544,56 @@ func testStreamPack(t *testing.T, version uint) { if length > len(data) { length = len(data) } + if shortFirstLoad { + length /= 2 + shortFirstLoad = false + } + + return data[:length] + } + + load := func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + data := loadBytes(length, offset) + if shortFirstLoad { + data = data[:len(data)/2] + shortFirstLoad = false + } - data = data[:length] loadCalls++ - return fn(bytes.NewReader(data)) + err := fn(bytes.NewReader(data)) + if err == nil { + return nil + } + var permanent *backoff.PermanentError + if errors.As(err, &permanent) { + return err + } + // retry loading once + return fn(bytes.NewReader(loadBytes(length, offset))) } // first, test regular usage t.Run("regular", func(t *testing.T) { tests := []struct { - blobs []restic.Blob - calls int + blobs []restic.Blob + calls int + shortFirstLoad bool }{ - {packfileBlobs[1:2], 1}, - {packfileBlobs[2:5], 1}, - {packfileBlobs[2:8], 1}, + {packfileBlobs[1:2], 1, false}, + {packfileBlobs[2:5], 1, false}, + {packfileBlobs[2:8], 1, false}, {[]restic.Blob{ packfileBlobs[0], packfileBlobs[4], packfileBlobs[2], - }, 1}, + }, 1, false}, {[]restic.Blob{ packfileBlobs[0], packfileBlobs[len(packfileBlobs)-1], - }, 2}, + }, 2, false}, + {packfileBlobs[:], 1, true}, } for _, test := range tests { @@ -592,6 +620,7 @@ func testStreamPack(t *testing.T, version uint) { } loadCalls = 0 + shortFirstLoad = test.shortFirstLoad err = repository.StreamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob) if err != nil { t.Fatal(err) @@ -604,6 +633,7 @@ func testStreamPack(t *testing.T, version uint) { }) } }) + shortFirstLoad = false // next, test invalid uses, which should return an error t.Run("invalid", func(t *testing.T) { From f2fafbffaa5835492e45a474229de450223e0b70 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 30 Dec 2023 22:39:26 +0100 Subject: [PATCH 019/158] restore: only report errors for blobs that actually failed to load Previously, errors would be reported for all blobs of a packfile that failed to stream. Now, only the not yet processed blobs are reported. --- internal/restorer/filerestorer.go | 17 +++++++++- internal/restorer/filerestorer_test.go | 44 ++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index 3bb7489ba..1fc74c7f0 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -246,7 +246,10 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { return err } + // track already processed blobs for precise error reporting + processedBlobs := restic.NewBlobSet() err := repository.StreamPack(ctx, r.packLoader, r.key, pack.id, blobList, func(h restic.BlobHandle, blobData []byte, err error) error { + processedBlobs.Insert(h) blob := blobs[h.ID] if err != nil { for file := range blob.files { @@ -292,7 +295,19 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { }) if err != nil { - for file := range pack.files { + // only report error for not yet processed blobs + affectedFiles := make(map[*fileInfo]struct{}) + for _, blob := range blobList { + if processedBlobs.Has(blob.BlobHandle) { + continue + } + blob := blobs[blob.ID] + for file := range blob.files { + affectedFiles[file] = struct{}{} + } + } + + for file := range affectedFiles { if errFile := sanitizeError(file, err); errFile != nil { return errFile } diff --git a/internal/restorer/filerestorer_test.go b/internal/restorer/filerestorer_test.go index e798f2b8b..7d35da19c 100644 --- a/internal/restorer/filerestorer_test.go +++ b/internal/restorer/filerestorer_test.go @@ -316,3 +316,47 @@ func testPartialDownloadError(t *testing.T, part int) { rtest.OK(t, err) verifyRestore(t, r, repo) } + +func TestFatalDownloadError(t *testing.T) { + tempdir := rtest.TempDir(t) + content := []TestFile{ + { + name: "file1", + blobs: []TestBlob{ + {"data1-1", "pack1"}, + {"data1-2", "pack1"}, + }, + }, + { + name: "file2", + blobs: []TestBlob{ + {"data2-1", "pack1"}, + {"data2-2", "pack1"}, + {"data2-3", "pack1"}, + }, + }} + + repo := newTestRepo(content) + + loader := repo.loader + repo.loader = func(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + // only return half the data to break file2 + return loader(ctx, h, length/2, offset, fn) + } + + r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false, nil) + r.files = repo.files + + var errors []string + r.Error = func(s string, e error) error { + // ignore errors as in the `restore` command + errors = append(errors, s) + return nil + } + + err := r.restoreFiles(context.TODO()) + rtest.OK(t, err) + + rtest.Assert(t, len(errors) == 1, "unexpected number of restore errors, expected: 1, got: %v", len(errors)) + rtest.Assert(t, errors[0] == "file2", "expected error for file2, got: %v", errors[0]) +} From 620518aec60be0e85c301631198877e06f47a28a Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 7 Jan 2024 11:06:42 +0100 Subject: [PATCH 020/158] add changelog for better restore error reporting --- changelog/unreleased/pull-4624 | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 changelog/unreleased/pull-4624 diff --git a/changelog/unreleased/pull-4624 b/changelog/unreleased/pull-4624 new file mode 100644 index 000000000..fbdbb1558 --- /dev/null +++ b/changelog/unreleased/pull-4624 @@ -0,0 +1,11 @@ +Bugfix: Correct restore progress information if an error occurs + +If an error occurred while restoring a snapshot, this could cause the restore +progress bar to show incorrect information. In addition, if a data file could +not be loaded completely, then errors would also be reported for some already +restored files. + +We have improved the error reporting of the restore command to be more accurate. + +https://github.com/restic/restic/pull/4624 +https://forum.restic.net/t/errors-restoring-with-restic-on-windows-server-s3/6943 From ce7db90e08d5836bab9d1854b4e646e64983155d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 8 Jan 2024 21:30:29 +0100 Subject: [PATCH 021/158] sync CI and go dependencies with master branch --- .github/workflows/docker.yml | 2 +- .github/workflows/tests.yml | 6 ++-- go.mod | 34 +++++++++--------- go.sum | 68 ++++++++++++++++++------------------ 4 files changed, 55 insertions(+), 55 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f483f5760..a19767849 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -25,7 +25,7 @@ jobs: uses: actions/checkout@v4 - name: Log in to the Container registry - uses: docker/login-action@b4bedf8053341df3b5a9f9e0f2cf4e79e27360c6 + uses: docker/login-action@3d58c274f17dffee475a5520cbe67f0a882c4dbb with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 291fab0f0..45681c6c5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -62,7 +62,7 @@ jobs: steps: - name: Set up Go ${{ matrix.go }} - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} @@ -226,7 +226,7 @@ jobs: steps: - name: Set up Go ${{ env.latest_go }} - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ env.latest_go }} @@ -244,7 +244,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Go ${{ env.latest_go }} - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ env.latest_go }} diff --git a/go.mod b/go.mod index 71927c7b7..f07fc10ee 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,8 @@ module github.com/restic/restic require ( - cloud.google.com/go/storage v1.33.0 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 + cloud.google.com/go/storage v1.34.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 github.com/Backblaze/blazer v0.6.1 @@ -13,8 +13,8 @@ require ( github.com/go-ole/go-ole v1.3.0 github.com/google/go-cmp v0.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 - github.com/klauspost/compress v1.17.2 - github.com/minio/minio-go/v7 v7.0.63 + github.com/klauspost/compress v1.17.4 + github.com/minio/minio-go/v7 v7.0.66 github.com/minio/sha256-simd v1.0.1 github.com/ncw/swift/v2 v2.0.2 github.com/pkg/errors v0.9.1 @@ -25,15 +25,15 @@ require ( github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 go.uber.org/automaxprocs v1.5.3 - golang.org/x/crypto v0.14.0 - golang.org/x/net v0.17.0 - golang.org/x/oauth2 v0.13.0 - golang.org/x/sync v0.4.0 - golang.org/x/sys v0.13.0 - golang.org/x/term v0.13.0 - golang.org/x/text v0.13.0 - golang.org/x/time v0.3.0 - google.golang.org/api v0.148.0 + golang.org/x/crypto v0.17.0 + golang.org/x/net v0.19.0 + golang.org/x/oauth2 v0.15.0 + golang.org/x/sync v0.5.0 + golang.org/x/sys v0.15.0 + golang.org/x/term v0.15.0 + golang.org/x/text v0.14.0 + golang.org/x/time v0.5.0 + google.golang.org/api v0.149.0 ) require ( @@ -41,7 +41,7 @@ require ( cloud.google.com/go/compute v1.23.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.3 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -51,12 +51,12 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect github.com/google/s2a-go v0.1.7 // indirect - github.com/google/uuid v1.3.1 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.1 // indirect + github.com/google/uuid v1.5.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/kr/fs v0.1.0 // indirect github.com/kr/text v0.2.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect diff --git a/go.sum b/go.sum index ab67783b2..01e58cccb 100644 --- a/go.sum +++ b/go.sum @@ -7,14 +7,14 @@ cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGB cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/iam v1.1.3 h1:18tKG7DzydKWUnLjonWcJO6wjSCAtzh4GcRKlH/Hrzc= cloud.google.com/go/iam v1.1.3/go.mod h1:3khUlaBXfPKKe7huYgEpDn6FtgRyMEqbkvBxrQyY5SE= -cloud.google.com/go/storage v1.33.0 h1:PVrDOkIC8qQVa1P3SXGpQvfuJhN2LHOoyZvWs8D2X5M= -cloud.google.com/go/storage v1.33.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/OxflYkiD8= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 h1:9kDVnTz3vbfweTqAUmk/a/pH5pWFCHtvRpHYC0G/dcA= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0/go.mod h1:3Ug6Qzto9anB6mGlEdgYMDF5zHQ+wwhEaYR4s17PHMw= +cloud.google.com/go/storage v1.34.0 h1:9KHBBTbaHPsNxO043SFmH3pMojjZiW+BFl9H41L7xjk= +cloud.google.com/go/storage v1.34.0/go.mod h1:Eji+S0CCQebjsiXxyIvPItC3BN3zWsdJjWfHfoLblgY= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.4.0 h1:TuEMD+E+1aTjjLICGQOW6vLe8UWES7kopac9mUXL56Y= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.4.0/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 h1:gggzg0SUMs6SQbEw+3LoSsYf9YMjkupeAnHMX8O9mmY= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0/go.mod h1:+6KLcKIVgxoBDMqMO/Nvy7bZ9a0nbU3I1DtFQK3YvB4= @@ -95,10 +95,10 @@ github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0Z github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.1 h1:SBWmZhjUDRorQxrN0nwzf+AHBxnbFjViHQS4P0yVpmQ= -github.com/googleapis/enterprise-certificate-proxy v0.3.1/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= @@ -108,11 +108,11 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= -github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= -github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= +github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -122,8 +122,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ= -github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4= +github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw= +github.com/minio/minio-go/v7 v7.0.66/go.mod h1:DHAgmyQEGdW3Cif0UooKOyrT3Vxs82zNdV6tkKhRtbs= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -183,8 +183,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -202,18 +202,18 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= -golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= +golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= +golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -229,22 +229,22 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= -golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -258,8 +258,8 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -google.golang.org/api v0.148.0 h1:HBq4TZlN4/1pNcu0geJZ/Q50vIwIXT532UIMYoo0vOs= -google.golang.org/api v0.148.0/go.mod h1:8/TBgwaKjfqTdacOJrOv2+2Q6fBDU1uHKK06oGSkxzU= +google.golang.org/api v0.149.0 h1:b2CqT6kG+zqJIVKRQ3ELJVLN1PwHZ6DJ3dW8yl82rgY= +google.golang.org/api v0.149.0/go.mod h1:Mwn1B7JTXrzXtnvmzQE2BD6bYZQ8DShKZDZbeN9I7qI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= From b856e9489a6e9256653aff2dc17d6254a96e2e7e Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 7 Jan 2024 12:00:32 +0100 Subject: [PATCH 022/158] restore: split downloadPack into smaller methods --- internal/restorer/filerestorer.go | 122 ++++++++++++++++-------------- 1 file changed, 65 insertions(+), 57 deletions(-) diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index 1fc74c7f0..7621e5ebb 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -197,12 +197,13 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { return wg.Wait() } -func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { +type blobToFileOffsetsMapping map[restic.ID]struct { + files map[*fileInfo][]int64 // file -> offsets (plural!) of the blob in the file +} +func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { // calculate blob->[]files->[]offsets mappings - blobs := make(map[restic.ID]struct { - files map[*fileInfo][]int64 // file -> offsets (plural!) of the blob in the file - }) + blobs := make(blobToFileOffsetsMapping) var blobList []restic.Blob for file := range pack.files { addBlob := func(blob restic.Blob, fileOffset int64) { @@ -239,60 +240,9 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { } } - sanitizeError := func(file *fileInfo, err error) error { - if err != nil { - err = r.Error(file.location, err) - } - return err - } - // track already processed blobs for precise error reporting processedBlobs := restic.NewBlobSet() - err := repository.StreamPack(ctx, r.packLoader, r.key, pack.id, blobList, func(h restic.BlobHandle, blobData []byte, err error) error { - processedBlobs.Insert(h) - blob := blobs[h.ID] - if err != nil { - for file := range blob.files { - if errFile := sanitizeError(file, err); errFile != nil { - return errFile - } - } - return nil - } - for file, offsets := range blob.files { - for _, offset := range offsets { - writeToFile := func() error { - // this looks overly complicated and needs explanation - // two competing requirements: - // - must create the file once and only once - // - should allow concurrent writes to the file - // so write the first blob while holding file lock - // write other blobs after releasing the lock - createSize := int64(-1) - file.lock.Lock() - if file.inProgress { - file.lock.Unlock() - } else { - defer file.lock.Unlock() - file.inProgress = true - createSize = file.size - } - writeErr := r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize, file.sparse) - - if r.progress != nil { - r.progress.AddProgress(file.location, uint64(len(blobData)), uint64(file.size)) - } - - return writeErr - } - err := sanitizeError(file, writeToFile()) - if err != nil { - return err - } - } - } - return nil - }) + err := r.downloadBlobs(ctx, pack.id, blobList, blobs, processedBlobs) if err != nil { // only report error for not yet processed blobs @@ -308,7 +258,7 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { } for file := range affectedFiles { - if errFile := sanitizeError(file, err); errFile != nil { + if errFile := r.sanitizeError(file, err); errFile != nil { return errFile } } @@ -316,3 +266,61 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { return nil } + +func (r *fileRestorer) sanitizeError(file *fileInfo, err error) error { + if err != nil { + err = r.Error(file.location, err) + } + return err +} + +func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID, blobList []restic.Blob, + blobs blobToFileOffsetsMapping, processedBlobs restic.BlobSet) error { + + return repository.StreamPack(ctx, r.packLoader, r.key, packID, blobList, + func(h restic.BlobHandle, blobData []byte, err error) error { + processedBlobs.Insert(h) + blob := blobs[h.ID] + if err != nil { + for file := range blob.files { + if errFile := r.sanitizeError(file, err); errFile != nil { + return errFile + } + } + return nil + } + for file, offsets := range blob.files { + for _, offset := range offsets { + writeToFile := func() error { + // this looks overly complicated and needs explanation + // two competing requirements: + // - must create the file once and only once + // - should allow concurrent writes to the file + // so write the first blob while holding file lock + // write other blobs after releasing the lock + createSize := int64(-1) + file.lock.Lock() + if file.inProgress { + file.lock.Unlock() + } else { + defer file.lock.Unlock() + file.inProgress = true + createSize = file.size + } + writeErr := r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize, file.sparse) + + if r.progress != nil { + r.progress.AddProgress(file.location, uint64(len(blobData)), uint64(file.size)) + } + + return writeErr + } + err := r.sanitizeError(file, writeToFile()) + if err != nil { + return err + } + } + } + return nil + }) +} From cf352ccafb3ad44f9a896b2b6d03b26f777458c0 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 7 Jan 2024 12:06:36 +0100 Subject: [PATCH 023/158] restore: cleanup downloadPack --- internal/restorer/filerestorer.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index 7621e5ebb..403651763 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -199,18 +199,18 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { type blobToFileOffsetsMapping map[restic.ID]struct { files map[*fileInfo][]int64 // file -> offsets (plural!) of the blob in the file + blob restic.Blob } func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { // calculate blob->[]files->[]offsets mappings blobs := make(blobToFileOffsetsMapping) - var blobList []restic.Blob for file := range pack.files { addBlob := func(blob restic.Blob, fileOffset int64) { blobInfo, ok := blobs[blob.ID] if !ok { blobInfo.files = make(map[*fileInfo][]int64) - blobList = append(blobList, blob) + blobInfo.blob = blob blobs[blob.ID] = blobInfo } blobInfo.files[file] = append(blobInfo.files[file], fileOffset) @@ -242,17 +242,16 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { // track already processed blobs for precise error reporting processedBlobs := restic.NewBlobSet() - err := r.downloadBlobs(ctx, pack.id, blobList, blobs, processedBlobs) + err := r.downloadBlobs(ctx, pack.id, blobs, processedBlobs) if err != nil { // only report error for not yet processed blobs affectedFiles := make(map[*fileInfo]struct{}) - for _, blob := range blobList { - if processedBlobs.Has(blob.BlobHandle) { + for _, entry := range blobs { + if processedBlobs.Has(entry.blob.BlobHandle) { continue } - blob := blobs[blob.ID] - for file := range blob.files { + for file := range entry.files { affectedFiles[file] = struct{}{} } } @@ -274,9 +273,13 @@ func (r *fileRestorer) sanitizeError(file *fileInfo, err error) error { return err } -func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID, blobList []restic.Blob, +func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID, blobs blobToFileOffsetsMapping, processedBlobs restic.BlobSet) error { + blobList := make([]restic.Blob, 0, len(blobs)) + for _, entry := range blobs { + blobList = append(blobList, entry.blob) + } return repository.StreamPack(ctx, r.packLoader, r.key, packID, blobList, func(h restic.BlobHandle, blobData []byte, err error) error { processedBlobs.Insert(h) From f0f89d7f27f6460a692eef252cb7df0021e6d66a Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 7 Jan 2024 12:20:31 +0100 Subject: [PATCH 024/158] restore: split error reporting from downloadPack --- internal/restorer/filerestorer.go | 45 +++++++++++++++++-------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index 403651763..f2e2cf24a 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -244,26 +244,7 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { processedBlobs := restic.NewBlobSet() err := r.downloadBlobs(ctx, pack.id, blobs, processedBlobs) - if err != nil { - // only report error for not yet processed blobs - affectedFiles := make(map[*fileInfo]struct{}) - for _, entry := range blobs { - if processedBlobs.Has(entry.blob.BlobHandle) { - continue - } - for file := range entry.files { - affectedFiles[file] = struct{}{} - } - } - - for file := range affectedFiles { - if errFile := r.sanitizeError(file, err); errFile != nil { - return errFile - } - } - } - - return nil + return r.reportError(blobs, processedBlobs, err) } func (r *fileRestorer) sanitizeError(file *fileInfo, err error) error { @@ -273,6 +254,30 @@ func (r *fileRestorer) sanitizeError(file *fileInfo, err error) error { return err } +func (r *fileRestorer) reportError(blobs blobToFileOffsetsMapping, processedBlobs restic.BlobSet, err error) error { + if err == nil { + return nil + } + + // only report error for not yet processed blobs + affectedFiles := make(map[*fileInfo]struct{}) + for _, entry := range blobs { + if processedBlobs.Has(entry.blob.BlobHandle) { + continue + } + for file := range entry.files { + affectedFiles[file] = struct{}{} + } + } + + for file := range affectedFiles { + if errFile := r.sanitizeError(file, err); errFile != nil { + return errFile + } + } + return nil +} + func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID, blobs blobToFileOffsetsMapping, processedBlobs restic.BlobSet) error { From 103beb96bcf037b7a897bc92848c875debdb0908 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 7 Jan 2024 12:17:35 +0100 Subject: [PATCH 025/158] restore: separately restore blobs that are frequently referenced Writing these blobs to their files can take a long time and consequently cause the backend connection to time out. Avoid that by retrieving these blobs separately. --- internal/restorer/filerestorer.go | 27 +++++++++++++++++++++++++- internal/restorer/filerestorer_test.go | 21 ++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index f2e2cf24a..99a460321 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -242,8 +242,33 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { // track already processed blobs for precise error reporting processedBlobs := restic.NewBlobSet() - err := r.downloadBlobs(ctx, pack.id, blobs, processedBlobs) + for _, entry := range blobs { + occurrences := 0 + for _, offsets := range entry.files { + occurrences += len(offsets) + } + // With a maximum blob size of 8MB, the normal blob streaming has to write + // at most 800MB for a single blob. This should be short enough to avoid + // network connection timeouts. Based on a quick test, a limit of 100 only + // selects a very small number of blobs (the number of references per blob + // - aka. `count` - seem to follow a expontential distribution) + if occurrences > 100 { + // process frequently referenced blobs first as these can take a long time to write + // which can cause backend connections to time out + delete(blobs, entry.blob.ID) + partialBlobs := blobToFileOffsetsMapping{entry.blob.ID: entry} + err := r.downloadBlobs(ctx, pack.id, partialBlobs, processedBlobs) + if err := r.reportError(blobs, processedBlobs, err); err != nil { + return err + } + } + } + if len(blobs) == 0 { + return nil + } + + err := r.downloadBlobs(ctx, pack.id, blobs, processedBlobs) return r.reportError(blobs, processedBlobs, err) } diff --git a/internal/restorer/filerestorer_test.go b/internal/restorer/filerestorer_test.go index 7d35da19c..8d4e2d4d2 100644 --- a/internal/restorer/filerestorer_test.go +++ b/internal/restorer/filerestorer_test.go @@ -247,6 +247,27 @@ func TestFileRestorerPackSkip(t *testing.T) { } } +func TestFileRestorerFrequentBlob(t *testing.T) { + tempdir := rtest.TempDir(t) + + for _, sparse := range []bool{false, true} { + blobs := []TestBlob{ + {"data1-1", "pack1-1"}, + } + for i := 0; i < 10000; i++ { + blobs = append(blobs, TestBlob{"a", "pack1-1"}) + } + blobs = append(blobs, TestBlob{"end", "pack1-1"}) + + restoreAndVerify(t, tempdir, []TestFile{ + { + name: "file1", + blobs: blobs, + }, + }, nil, sparse) + } +} + func TestErrorRestoreFiles(t *testing.T) { tempdir := rtest.TempDir(t) content := []TestFile{ From ec0fb46f6c7f5a4331f35f6bdf9d53af255e8f8c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 7 Jan 2024 14:18:20 +0100 Subject: [PATCH 026/158] add changelog for reliable restores --- changelog/unreleased/pull-4626 | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 changelog/unreleased/pull-4626 diff --git a/changelog/unreleased/pull-4626 b/changelog/unreleased/pull-4626 new file mode 100644 index 000000000..ea16d749f --- /dev/null +++ b/changelog/unreleased/pull-4626 @@ -0,0 +1,11 @@ +Bugfix: Improve reliability of restoring large files + +In some cases restic failed to restore large files that frequently contain the +same file chunk. In combination with certain backends, this could result in +network connection timeouts that caused incomplete restores. + +Restic now includes special handling for such file chunks to ensure reliable +restores. + +https://github.com/restic/restic/pull/4626 +https://forum.restic.net/t/errors-restoring-with-restic-on-windows-server-s3/6943 From e4bbde7036ed965f4d2548a07e6c527a6de04003 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 17:31:16 +0100 Subject: [PATCH 027/158] rclone: Workaround for incorrect "not found" errors while listing files rclone returns a "not found" error if an internal error occurs while listing a folder. Ignoring this error lets restic erroneously think that there are no files, which can cause `prune` to wipe the whole repository. --- changelog/unreleased/issue-4612 | 11 +++++++++++ internal/backend/rest/rest.go | 9 +++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/issue-4612 diff --git a/changelog/unreleased/issue-4612 b/changelog/unreleased/issue-4612 new file mode 100644 index 000000000..ed99f4767 --- /dev/null +++ b/changelog/unreleased/issue-4612 @@ -0,0 +1,11 @@ +Bugfix: Improve error handling for `rclone` backend + +Since restic 0.16.0, if rclone encountered an error while listing files, +this could in rare circumstances cause restic to assume that there are no +files. Although unlikely, this situation could result in data loss if it +were to happen right when the `prune` command is listing existing snapshots. + +Error handling has now been improved to detect and work around this case. + +https://github.com/restic/restic/issues/4612 +https://github.com/restic/restic/pull/4618 diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index f8670280d..de730e21f 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -328,8 +328,13 @@ func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(restic.Fi } if resp.StatusCode == http.StatusNotFound { - // ignore missing directories - return nil + if !strings.HasPrefix(resp.Header.Get("Server"), "rclone/") { + // ignore missing directories, unless the server is rclone. rclone + // already ignores missing directories, but misuses "not found" to + // report certain internal errors, see + // https://github.com/rclone/rclone/pull/7550 for details. + return nil + } } if resp.StatusCode != http.StatusOK { From ef2e473b99e48397a5b32b50c534776cfe5f4249 Mon Sep 17 00:00:00 2001 From: "Leo R. Lundgren" Date: Wed, 10 Jan 2024 00:18:32 +0100 Subject: [PATCH 028/158] doc: Polish changelogs --- changelog/unreleased/issue-4560 | 2 +- changelog/unreleased/issue-4574 | 10 +++++----- changelog/unreleased/pull-4624 | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/changelog/unreleased/issue-4560 b/changelog/unreleased/issue-4560 index c421f6e69..4346019d5 100644 --- a/changelog/unreleased/issue-4560 +++ b/changelog/unreleased/issue-4560 @@ -4,7 +4,7 @@ Since Go 1.21, most filesystem reparse points on Windows are considered to be irregular files. This caused restic to show an `error: invalid node type ""` error message for those files. -We have improved the error message to include the file path for those files: +This error message has now been improved and includes the relevant file path: `error: nodeFromFileInfo path/to/file: unsupported file type "irregular"`. As irregular files are not required to behave like regular files, it is not possible to provide a generic way to back up those files. diff --git a/changelog/unreleased/issue-4574 b/changelog/unreleased/issue-4574 index 3668ae6c3..bfb41620e 100644 --- a/changelog/unreleased/issue-4574 +++ b/changelog/unreleased/issue-4574 @@ -1,11 +1,11 @@ -Bugfix: support backup of deduplicated files on Windows again +Bugfix: Support backup of deduplicated files on Windows again With the official release builds of restic 0.16.1 and 0.16.2, it was not -possible to back up files that were deduplicated by the corresponding Windows -Server feature. This also applies to restic versions built using Go -1.21.0 - 1.21.4. +possible to back up files that were deduplicated by the corresponding +Windows Server feature. This also applied to restic versions built using +Go 1.21.0-1.21.4. -We have updated the used Go version to fix this. +The Go version used to build restic has now been updated to fix this. https://github.com/restic/restic/issues/4574 https://github.com/restic/restic/pull/4621 diff --git a/changelog/unreleased/pull-4624 b/changelog/unreleased/pull-4624 index fbdbb1558..6fff3c9f9 100644 --- a/changelog/unreleased/pull-4624 +++ b/changelog/unreleased/pull-4624 @@ -1,11 +1,11 @@ -Bugfix: Correct restore progress information if an error occurs +Bugfix: Correct `restore` progress information if an error occurs -If an error occurred while restoring a snapshot, this could cause the restore +If an error occurred while restoring a snapshot, this could cause the `restore` progress bar to show incorrect information. In addition, if a data file could not be loaded completely, then errors would also be reported for some already restored files. -We have improved the error reporting of the restore command to be more accurate. +Error reporting of the `restore` command has now been made more accurate. https://github.com/restic/restic/pull/4624 https://forum.restic.net/t/errors-restoring-with-restic-on-windows-server-s3/6943 From d987582594afbdc95c2948890f136c1d1050aa2e Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 14 Jan 2024 20:21:19 +0100 Subject: [PATCH 029/158] Prepare changelog for 0.16.3 --- changelog/{unreleased => 0.16.3_2024-01-14}/issue-4560 | 0 changelog/{unreleased => 0.16.3_2024-01-14}/issue-4574 | 0 changelog/{unreleased => 0.16.3_2024-01-14}/issue-4612 | 0 changelog/{unreleased => 0.16.3_2024-01-14}/pull-4624 | 0 changelog/{unreleased => 0.16.3_2024-01-14}/pull-4626 | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename changelog/{unreleased => 0.16.3_2024-01-14}/issue-4560 (100%) rename changelog/{unreleased => 0.16.3_2024-01-14}/issue-4574 (100%) rename changelog/{unreleased => 0.16.3_2024-01-14}/issue-4612 (100%) rename changelog/{unreleased => 0.16.3_2024-01-14}/pull-4624 (100%) rename changelog/{unreleased => 0.16.3_2024-01-14}/pull-4626 (100%) diff --git a/changelog/unreleased/issue-4560 b/changelog/0.16.3_2024-01-14/issue-4560 similarity index 100% rename from changelog/unreleased/issue-4560 rename to changelog/0.16.3_2024-01-14/issue-4560 diff --git a/changelog/unreleased/issue-4574 b/changelog/0.16.3_2024-01-14/issue-4574 similarity index 100% rename from changelog/unreleased/issue-4574 rename to changelog/0.16.3_2024-01-14/issue-4574 diff --git a/changelog/unreleased/issue-4612 b/changelog/0.16.3_2024-01-14/issue-4612 similarity index 100% rename from changelog/unreleased/issue-4612 rename to changelog/0.16.3_2024-01-14/issue-4612 diff --git a/changelog/unreleased/pull-4624 b/changelog/0.16.3_2024-01-14/pull-4624 similarity index 100% rename from changelog/unreleased/pull-4624 rename to changelog/0.16.3_2024-01-14/pull-4624 diff --git a/changelog/unreleased/pull-4626 b/changelog/0.16.3_2024-01-14/pull-4626 similarity index 100% rename from changelog/unreleased/pull-4626 rename to changelog/0.16.3_2024-01-14/pull-4626 From b70b94507a6df3645607827316520616dbf3369d Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 14 Jan 2024 20:21:19 +0100 Subject: [PATCH 030/158] Generate CHANGELOG.md for 0.16.3 --- CHANGELOG.md | 3418 +++++++++++++++++++++++++++----------------------- 1 file changed, 1859 insertions(+), 1559 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 872b70a1b..ba6fa7a11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # Table of Contents +* [Changelog for 0.16.3](#changelog-for-restic-0163-2024-01-14) * [Changelog for 0.16.2](#changelog-for-restic-0162-2023-10-29) * [Changelog for 0.16.1](#changelog-for-restic-0161-2023-10-24) * [Changelog for 0.16.0](#changelog-for-restic-0160-2023-07-31) @@ -31,6 +32,84 @@ * [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29) +# Changelog for restic 0.16.3 (2024-01-14) +The following sections list the changes in restic 0.16.3 relevant to +restic users. The changes are ordered by importance. + +## Summary + + * Fix #4560: Improve errors for irregular files on Windows + * Fix #4574: Support backup of deduplicated files on Windows again + * Fix #4612: Improve error handling for `rclone` backend + * Fix #4624: Correct `restore` progress information if an error occurs + * Fix #4626: Improve reliability of restoring large files + +## Details + + * Bugfix #4560: Improve errors for irregular files on Windows + + Since Go 1.21, most filesystem reparse points on Windows are considered to be + irregular files. This caused restic to show an `error: invalid node type ""` + error message for those files. + + This error message has now been improved and includes the relevant file path: + `error: nodeFromFileInfo path/to/file: unsupported file type "irregular"`. As + irregular files are not required to behave like regular files, it is not + possible to provide a generic way to back up those files. + + https://github.com/restic/restic/issues/4560 + https://github.com/restic/restic/pull/4620 + https://forum.restic.net/t/windows-backup-error-invalid-node-type/6875 + + * Bugfix #4574: Support backup of deduplicated files on Windows again + + With the official release builds of restic 0.16.1 and 0.16.2, it was not + possible to back up files that were deduplicated by the corresponding Windows + Server feature. This also applied to restic versions built using Go + 1.21.0-1.21.4. + + The Go version used to build restic has now been updated to fix this. + + https://github.com/restic/restic/issues/4574 + https://github.com/restic/restic/pull/4621 + + * Bugfix #4612: Improve error handling for `rclone` backend + + Since restic 0.16.0, if rclone encountered an error while listing files, this + could in rare circumstances cause restic to assume that there are no files. + Although unlikely, this situation could result in data loss if it were to happen + right when the `prune` command is listing existing snapshots. + + Error handling has now been improved to detect and work around this case. + + https://github.com/restic/restic/issues/4612 + https://github.com/restic/restic/pull/4618 + + * Bugfix #4624: Correct `restore` progress information if an error occurs + + If an error occurred while restoring a snapshot, this could cause the `restore` + progress bar to show incorrect information. In addition, if a data file could + not be loaded completely, then errors would also be reported for some already + restored files. + + Error reporting of the `restore` command has now been made more accurate. + + https://github.com/restic/restic/pull/4624 + https://forum.restic.net/t/errors-restoring-with-restic-on-windows-server-s3/6943 + + * Bugfix #4626: Improve reliability of restoring large files + + In some cases restic failed to restore large files that frequently contain the + same file chunk. In combination with certain backends, this could result in + network connection timeouts that caused incomplete restores. + + Restic now includes special handling for such file chunks to ensure reliable + restores. + + https://github.com/restic/restic/pull/4626 + https://forum.restic.net/t/errors-restoring-with-restic-on-windows-server-s3/6943 + + # Changelog for restic 0.16.2 (2023-10-29) The following sections list the changes in restic 0.16.2 relevant to restic users. The changes are ordered by importance. @@ -44,16 +123,18 @@ restic users. The changes are ordered by importance. * Bugfix #4540: Restore ARMv5 support for ARM binaries - The official release binaries for restic 0.16.1 were accidentally built to require ARMv7. The - build process is now updated to restore support for ARMv5. + The official release binaries for restic 0.16.1 were accidentally built to + require ARMv7. The build process is now updated to restore support for ARMv5. - Please note that restic 0.17.0 will drop support for ARMv5 and require at least ARMv6. + Please note that restic 0.17.0 will drop support for ARMv5 and require at least + ARMv6. https://github.com/restic/restic/issues/4540 * Bugfix #4545: Repair documentation build on Read the Docs - For restic 0.16.1, no documentation was available at https://restic.readthedocs.io/ . + For restic 0.16.1, no documentation was available at + https://restic.readthedocs.io/ . The documentation build process is now updated to work again. @@ -80,65 +161,67 @@ restic users. The changes are ordered by importance. * Bugfix #4513: Make `key list` command honor `--no-lock` - The `key list` command now supports the `--no-lock` options. This allows determining which - keys a repo can be accessed by without the need for having write access (e.g., read-only sftp - access, filesystem snapshot). + The `key list` command now supports the `--no-lock` options. This allows + determining which keys a repo can be accessed by without the need for having + write access (e.g., read-only sftp access, filesystem snapshot). https://github.com/restic/restic/issues/4513 https://github.com/restic/restic/pull/4514 * Bugfix #4516: Do not try to load password on command line autocomplete - The command line autocompletion previously tried to load the repository password. This could - cause the autocompletion not to work. Now, this step gets skipped. + The command line autocompletion previously tried to load the repository + password. This could cause the autocompletion not to work. Now, this step gets + skipped. https://github.com/restic/restic/issues/4516 https://github.com/restic/restic/pull/4526 * Bugfix #4523: Update zstd library to fix possible data corruption at max. compression - In restic 0.16.0, backups where the compression level was set to `max` (using `--compression - max`) could in rare and very specific circumstances result in data corruption due to a bug in the - library used for compressing data. + In restic 0.16.0, backups where the compression level was set to `max` (using + `--compression max`) could in rare and very specific circumstances result in + data corruption due to a bug in the library used for compressing data. - Restic now uses the latest version of the library used to compress data, which includes a fix for - this issue. Please note that the `auto` compression level (which restic uses by default) was - never affected, and even if you used `max` compression, chances of being affected by this issue - were very small. + Restic now uses the latest version of the library used to compress data, which + includes a fix for this issue. Please note that the `auto` compression level + (which restic uses by default) was never affected, and even if you used `max` + compression, chances of being affected by this issue were very small. - To check a repository for any corruption, run `restic check --read-data`. This will download - and verify the whole repository and can be used at any time to completely verify the integrity of - a repository. If the `check` command detects anomalies, follow the suggested steps. + To check a repository for any corruption, run `restic check --read-data`. This + will download and verify the whole repository and can be used at any time to + completely verify the integrity of a repository. If the `check` command detects + anomalies, follow the suggested steps. - To simplify any needed repository repair and minimize data loss, there is also a new and - experimental `repair packs` command that salvages all valid data from the affected pack files - (see `restic help repair packs` for more information). + To simplify any needed repository repair and minimize data loss, there is also a + new and experimental `repair packs` command that salvages all valid data from + the affected pack files (see `restic help repair packs` for more information). https://github.com/restic/restic/issues/4523 https://github.com/restic/restic/pull/4530 * Change #4532: Update dependencies and require Go 1.19 or newer - We have updated all dependencies. Since some libraries require newer Go standard library - features, support for Go 1.18 has been dropped, which means that restic now requires at least Go - 1.19 to build. + We have updated all dependencies. Since some libraries require newer Go standard + library features, support for Go 1.18 has been dropped, which means that restic + now requires at least Go 1.19 to build. https://github.com/restic/restic/pull/4532 https://github.com/restic/restic/pull/4533 * Enhancement #229: Show progress bar while loading the index - Restic did not provide any feedback while loading index files. Now, there is a progress bar that - shows the index loading progress. + Restic did not provide any feedback while loading index files. Now, there is a + progress bar that shows the index loading progress. https://github.com/restic/restic/issues/229 https://github.com/restic/restic/pull/4419 * Enhancement #4128: Automatically set `GOMAXPROCS` in resource-constrained containers - When running restic in a Linux container with CPU-usage limits, restic now automatically - adjusts `GOMAXPROCS`. This helps to reduce the memory consumption on hosts with many CPU - cores. + When running restic in a Linux container with CPU-usage limits, restic now + automatically adjusts `GOMAXPROCS`. This helps to reduce the memory consumption + on hosts with many CPU cores. https://github.com/restic/restic/issues/4128 https://github.com/restic/restic/pull/4485 @@ -146,32 +229,33 @@ restic users. The changes are ordered by importance. * Enhancement #4480: Allow setting REST password and username via environment variables - Previously, it was only possible to specify the REST-server username and password in the - repository URL, or by using the `--repository-file` option. This meant it was not possible to - use authentication in contexts where the repository URL is stored in publicly accessible way. + Previously, it was only possible to specify the REST-server username and + password in the repository URL, or by using the `--repository-file` option. This + meant it was not possible to use authentication in contexts where the repository + URL is stored in publicly accessible way. - Restic now allows setting the username and password using the `RESTIC_REST_USERNAME` and - `RESTIC_REST_PASSWORD` variables. + Restic now allows setting the username and password using the + `RESTIC_REST_USERNAME` and `RESTIC_REST_PASSWORD` variables. https://github.com/restic/restic/pull/4480 * Enhancement #4511: Include inode numbers in JSON output for `find` and `ls` commands - Restic used to omit the inode numbers in the JSON messages emitted for nodes by the `ls` command - as well as for matches by the `find` command. It now includes those values whenever they are - available. + Restic used to omit the inode numbers in the JSON messages emitted for nodes by + the `ls` command as well as for matches by the `find` command. It now includes + those values whenever they are available. https://github.com/restic/restic/pull/4511 * Enhancement #4519: Add config option to set SFTP command arguments - When using the `sftp` backend, scenarios where a custom identity file was needed for the SSH - connection, required the full command to be specified: `-o sftp.command='ssh - user@host:port -i /ssh/my_private_key -s sftp'` + When using the `sftp` backend, scenarios where a custom identity file was needed + for the SSH connection, required the full command to be specified: `-o + sftp.command='ssh user@host:port -i /ssh/my_private_key -s sftp'` - Now, the `-o sftp.args=...` option can be passed to restic to specify custom arguments for the - SSH command executed by the SFTP backend. This simplifies the above example to `-o - sftp.args='-i /ssh/my_private_key'`. + Now, the `-o sftp.args=...` option can be passed to restic to specify custom + arguments for the SSH command executed by the SFTP backend. This simplifies the + above example to `-o sftp.args='-i /ssh/my_private_key'`. https://github.com/restic/restic/issues/4241 https://github.com/restic/restic/pull/4519 @@ -217,31 +301,32 @@ restic users. The changes are ordered by importance. * Bugfix #2565: Support "unlimited" in `forget --keep-*` options - Restic would previously forget snapshots that should have been kept when a negative value was - passed to the `--keep-*` options. Negative values are now forbidden. To keep all snapshots, - the special value `unlimited` is now supported. For example, `--keep-monthly unlimited` - will keep all monthly snapshots. + Restic would previously forget snapshots that should have been kept when a + negative value was passed to the `--keep-*` options. Negative values are now + forbidden. To keep all snapshots, the special value `unlimited` is now + supported. For example, `--keep-monthly unlimited` will keep all monthly + snapshots. https://github.com/restic/restic/issues/2565 https://github.com/restic/restic/pull/4234 * Bugfix #3311: Support non-UTF8 paths as symlink target - Earlier restic versions did not correctly `backup` and `restore` symlinks that contain a - non-UTF8 target. Note that this only affected systems that still use a non-Unicode encoding - for filesystem paths. + Earlier restic versions did not correctly `backup` and `restore` symlinks that + contain a non-UTF8 target. Note that this only affected systems that still use a + non-Unicode encoding for filesystem paths. - The repository format is now extended to add support for such symlinks. Please note that - snapshots must have been created with at least restic version 0.16.0 for `restore` to - correctly handle non-UTF8 symlink targets when restoring them. + The repository format is now extended to add support for such symlinks. Please + note that snapshots must have been created with at least restic version 0.16.0 + for `restore` to correctly handle non-UTF8 symlink targets when restoring them. https://github.com/restic/restic/issues/3311 https://github.com/restic/restic/pull/3802 * Bugfix #4199: Avoid lock refresh issues on slow network connections - On network connections with a low upload speed, backups and other operations could fail with - the error message `Fatal: failed to refresh lock in time`. + On network connections with a low upload speed, backups and other operations + could fail with the error message `Fatal: failed to refresh lock in time`. This has now been fixed by reworking the lock refresh handling. @@ -250,21 +335,21 @@ restic users. The changes are ordered by importance. * Bugfix #4274: Improve lock refresh handling after standby - If the restic process was stopped or the host running restic entered standby during a long - running operation such as a backup, this previously resulted in the operation failing with - `Fatal: failed to refresh lock in time`. + If the restic process was stopped or the host running restic entered standby + during a long running operation such as a backup, this previously resulted in + the operation failing with `Fatal: failed to refresh lock in time`. - This has now been fixed such that restic first checks whether it is safe to continue the current - operation and only throws an error if not. + This has now been fixed such that restic first checks whether it is safe to + continue the current operation and only throws an error if not. https://github.com/restic/restic/issues/4274 https://github.com/restic/restic/pull/4374 * Bugfix #4319: Correctly clean up status bar output of the `backup` command - Due to a regression in restic 0.15.2, the status bar of the `backup` command could leave some - output behind. This happened if filenames were printed that are wider than the current - terminal width. This has now been fixed. + Due to a regression in restic 0.15.2, the status bar of the `backup` command + could leave some output behind. This happened if filenames were printed that are + wider than the current terminal width. This has now been fixed. https://github.com/restic/restic/issues/4319 https://github.com/restic/restic/pull/4318 @@ -275,25 +360,26 @@ restic users. The changes are ordered by importance. * Bugfix #4400: Ignore missing folders in `rest` backend - If a repository accessed via the REST backend was missing folders, then restic would fail with - an error while trying to list the data in the repository. This has been now fixed. + If a repository accessed via the REST backend was missing folders, then restic + would fail with an error while trying to list the data in the repository. This + has been now fixed. https://github.com/restic/rest-server/issues/235 https://github.com/restic/restic/pull/4400 * Change #4176: Fix JSON message type of `scan_finished` for the `backup` command - Restic incorrectly set the `message_type` of the `scan_finished` message to `status` - instead of `verbose_status`. This has now been corrected so that the messages report the - correct type. + Restic incorrectly set the `message_type` of the `scan_finished` message to + `status` instead of `verbose_status`. This has now been corrected so that the + messages report the correct type. https://github.com/restic/restic/pull/4176 * Change #4201: Require Go 1.20 for Solaris builds - Building restic on Solaris now requires Go 1.20, as the library used to access Azure uses the - mmap syscall, which is only available on Solaris starting from Go 1.20. All other platforms - however continue to build with Go 1.18. + Building restic on Solaris now requires Go 1.20, as the library used to access + Azure uses the mmap syscall, which is only available on Solaris starting from Go + 1.20. All other platforms however continue to build with Go 1.18. https://github.com/restic/restic/pull/4201 @@ -314,8 +400,8 @@ restic users. The changes are ordered by importance. * Enhancement #719: Add `--retry-lock` option - This option allows specifying a duration for which restic will wait if the repository is - already locked. + This option allows specifying a duration for which restic will wait if the + repository is already locked. https://github.com/restic/restic/issues/719 https://github.com/restic/restic/pull/2214 @@ -323,24 +409,25 @@ restic users. The changes are ordered by importance. * Enhancement #1495: Sort snapshots by timestamp in `restic find` - The `find` command used to print snapshots in an arbitrary order. Restic now prints snapshots - sorted by timestamp. + The `find` command used to print snapshots in an arbitrary order. Restic now + prints snapshots sorted by timestamp. https://github.com/restic/restic/issues/1495 https://github.com/restic/restic/pull/4409 * Enhancement #1759: Add `repair index` and `repair snapshots` commands - The `rebuild-index` command has been renamed to `repair index`. The old name will still work, - but is deprecated. + The `rebuild-index` command has been renamed to `repair index`. The old name + will still work, but is deprecated. - When a snapshot was damaged, the only option up to now was to completely forget the snapshot, - even if only some unimportant files in it were damaged and other files were still fine. + When a snapshot was damaged, the only option up to now was to completely forget + the snapshot, even if only some unimportant files in it were damaged and other + files were still fine. - Restic now has a `repair snapshots` command, which can salvage any non-damaged files and parts - of files in the snapshots by removing damaged directories and missing file contents. Please - note that the damaged data may still be lost and see the "Troubleshooting" section in the - documentation for more details. + Restic now has a `repair snapshots` command, which can salvage any non-damaged + files and parts of files in the snapshots by removing damaged directories and + missing file contents. Please note that the damaged data may still be lost and + see the "Troubleshooting" section in the documentation for more details. https://github.com/restic/restic/issues/1759 https://github.com/restic/restic/issues/1714 @@ -352,19 +439,20 @@ restic users. The changes are ordered by importance. * Enhancement #1926: Allow certificate paths to be passed through environment variables - Restic will now read paths to certificates from the environment variables `RESTIC_CACERT` or - `RESTIC_TLS_CLIENT_CERT` if `--cacert` or `--tls-client-cert` are not specified. + Restic will now read paths to certificates from the environment variables + `RESTIC_CACERT` or `RESTIC_TLS_CLIENT_CERT` if `--cacert` or `--tls-client-cert` + are not specified. https://github.com/restic/restic/issues/1926 https://github.com/restic/restic/pull/4384 * Enhancement #2359: Provide multi-platform Docker images - The official Docker images are now built for the architectures linux/386, linux/amd64, - linux/arm and linux/arm64. + The official Docker images are now built for the architectures linux/386, + linux/amd64, linux/arm and linux/arm64. - As an alternative to the Docker Hub, the Docker images are also available on ghcr.io, the GitHub - Container Registry. + As an alternative to the Docker Hub, the Docker images are also available on + ghcr.io, the GitHub Container Registry. https://github.com/restic/restic/issues/2359 https://github.com/restic/restic/issues/4269 @@ -374,25 +462,26 @@ restic users. The changes are ordered by importance. The `azure` backend previously only supported storages using the global domain `core.windows.net`. This meant that backups to other domains such as Azure China - (`core.chinacloudapi.cn`) or Azure Germany (`core.cloudapi.de`) were not supported. - Restic now allows overriding the global domain using the environment variable - `AZURE_ENDPOINT_SUFFIX`. + (`core.chinacloudapi.cn`) or Azure Germany (`core.cloudapi.de`) were not + supported. Restic now allows overriding the global domain using the environment + variable `AZURE_ENDPOINT_SUFFIX`. https://github.com/restic/restic/issues/2468 https://github.com/restic/restic/pull/4387 * Enhancement #2679: Reduce file fragmentation for local backend - Before this change, local backend files could become fragmented. Now restic will try to - preallocate space for pack files to avoid their fragmentation. + Before this change, local backend files could become fragmented. Now restic will + try to preallocate space for pack files to avoid their fragmentation. https://github.com/restic/restic/issues/2679 https://github.com/restic/restic/pull/3261 * Enhancement #3328: Reduce memory usage by up to 25% - The in-memory index has been optimized to be more garbage collection friendly. Restic now - defaults to `GOGC=50` to run the Go garbage collector more frequently. + The in-memory index has been optimized to be more garbage collection friendly. + Restic now defaults to `GOGC=50` to run the Go garbage collector more + frequently. https://github.com/restic/restic/issues/3328 https://github.com/restic/restic/pull/4352 @@ -400,21 +489,21 @@ restic users. The changes are ordered by importance. * Enhancement #3397: Improve accuracy of ETA displayed during backup - Restic's `backup` command displayed an ETA that did not adapt when the rate of progress made - during the backup changed during the course of the backup. + Restic's `backup` command displayed an ETA that did not adapt when the rate of + progress made during the backup changed during the course of the backup. - Restic now uses recent progress when computing the ETA. It is important to realize that the - estimate may still be wrong, because restic cannot predict the future, but the hope is that the - ETA will be more accurate in most cases. + Restic now uses recent progress when computing the ETA. It is important to + realize that the estimate may still be wrong, because restic cannot predict the + future, but the hope is that the ETA will be more accurate in most cases. https://github.com/restic/restic/issues/3397 https://github.com/restic/restic/pull/3563 * Enhancement #3624: Keep oldest snapshot when there are not enough snapshots - The `forget` command now additionally preserves the oldest snapshot if fewer snapshots than - allowed by the `--keep-*` parameters would otherwise be kept. This maximizes the amount of - history kept within the specified limits. + The `forget` command now additionally preserves the oldest snapshot if fewer + snapshots than allowed by the `--keep-*` parameters would otherwise be kept. + This maximizes the amount of history kept within the specified limits. https://github.com/restic/restic/issues/3624 https://github.com/restic/restic/pull/4366 @@ -422,99 +511,106 @@ restic users. The changes are ordered by importance. * Enhancement #3698: Add support for Managed / Workload Identity to `azure` backend - Restic now additionally supports authenticating to Azure using Workload Identity or Managed - Identity credentials, which are automatically injected in several environments such as a - managed Kubernetes cluster. + Restic now additionally supports authenticating to Azure using Workload Identity + or Managed Identity credentials, which are automatically injected in several + environments such as a managed Kubernetes cluster. https://github.com/restic/restic/issues/3698 https://github.com/restic/restic/pull/4029 * Enhancement #3871: Support `:` syntax to select subfolders - Commands like `diff` or `restore` always worked with the full snapshot. This did not allow - comparing only a specific subfolder or only restoring that folder (`restore --include - subfolder` filters the restored files, but still creates the directories included in - `subfolder`). + Commands like `diff` or `restore` always worked with the full snapshot. This did + not allow comparing only a specific subfolder or only restoring that folder + (`restore --include subfolder` filters the restored files, but still creates the + directories included in `subfolder`). - The commands `diff`, `dump`, `ls` and `restore` now support the `:` - syntax, where `snapshot` is the ID of a snapshot (or the string `latest`) and `subfolder` is a - path within the snapshot. The commands will then only work with the specified path of the - snapshot. The `subfolder` must be a path to a folder as returned by `ls`. Two examples: + The commands `diff`, `dump`, `ls` and `restore` now support the + `:` syntax, where `snapshot` is the ID of a snapshot (or + the string `latest`) and `subfolder` is a path within the snapshot. The commands + will then only work with the specified path of the snapshot. The `subfolder` + must be a path to a folder as returned by `ls`. Two examples: `restic restore -t target latest:/some/path` `restic diff 12345678:/some/path 90abcef:/some/path` - For debugging purposes, the `cat` command now supports `cat tree :` to - return the directory metadata for the given subfolder. + For debugging purposes, the `cat` command now supports `cat tree + :` to return the directory metadata for the given + subfolder. https://github.com/restic/restic/issues/3871 https://github.com/restic/restic/pull/4334 * Enhancement #3941: Support `--group-by` for backup parent selection - Previously, the `backup` command by default selected the parent snapshot based on the - hostname and the backup targets. When the backup path list changed, the `backup` command was - unable to determine a suitable parent snapshot and had to read all files again. + Previously, the `backup` command by default selected the parent snapshot based + on the hostname and the backup targets. When the backup path list changed, the + `backup` command was unable to determine a suitable parent snapshot and had to + read all files again. - The new `--group-by` option for the `backup` command allows filtering snapshots for the - parent selection by `host`, `paths` and `tags`. It defaults to `host,paths` which selects the - latest snapshot with hostname and paths matching those of the backup run. This matches the - behavior of prior restic versions. + The new `--group-by` option for the `backup` command allows filtering snapshots + for the parent selection by `host`, `paths` and `tags`. It defaults to + `host,paths` which selects the latest snapshot with hostname and paths matching + those of the backup run. This matches the behavior of prior restic versions. - The new `--group-by` option should be set to the same value as passed to `forget --group-by`. + The new `--group-by` option should be set to the same value as passed to `forget + --group-by`. https://github.com/restic/restic/issues/3941 https://github.com/restic/restic/pull/4081 * Enhancement #4130: Cancel current command if cache becomes unusable - If the cache directory was removed or ran out of space while restic was running, this would - previously cause further caching attempts to fail and thereby drastically slow down the - command execution. Now, the currently running command is instead canceled. + If the cache directory was removed or ran out of space while restic was running, + this would previously cause further caching attempts to fail and thereby + drastically slow down the command execution. Now, the currently running command + is instead canceled. https://github.com/restic/restic/issues/4130 https://github.com/restic/restic/pull/4166 * Enhancement #4159: Add `--human-readable` option to `ls` and `find` commands - Previously, when using the `-l` option with the `ls` and `find` commands, the displayed size - was always in bytes, without an option for a more human readable format such as MiB or GiB. + Previously, when using the `-l` option with the `ls` and `find` commands, the + displayed size was always in bytes, without an option for a more human readable + format such as MiB or GiB. - The new `--human-readable` option will convert longer size values into more human friendly - values with an appropriate suffix depending on the output size. For example, a size of - `14680064` will be shown as `14.000 MiB`. + The new `--human-readable` option will convert longer size values into more + human friendly values with an appropriate suffix depending on the output size. + For example, a size of `14680064` will be shown as `14.000 MiB`. https://github.com/restic/restic/issues/4159 https://github.com/restic/restic/pull/4351 * Enhancement #4188: Include restic version in snapshot metadata - The restic version used to backup a snapshot is now included in its metadata and shown when - inspecting a snapshot using `restic cat snapshot ` or `restic snapshots - --json`. + The restic version used to backup a snapshot is now included in its metadata and + shown when inspecting a snapshot using `restic cat snapshot ` or + `restic snapshots --json`. https://github.com/restic/restic/issues/4188 https://github.com/restic/restic/pull/4378 * Enhancement #4220: Add `jq` binary to Docker image - The Docker image now contains `jq`, which can be useful to process JSON data output by restic. + The Docker image now contains `jq`, which can be useful to process JSON data + output by restic. https://github.com/restic/restic/pull/4220 * Enhancement #4226: Allow specifying region of new buckets in the `gs` backend - Previously, buckets used by the Google Cloud Storage backend would always get created in the - "us" region. It is now possible to specify the region where a bucket should be created by using - the `-o gs.region=us` option. + Previously, buckets used by the Google Cloud Storage backend would always get + created in the "us" region. It is now possible to specify the region where a + bucket should be created by using the `-o gs.region=us` option. https://github.com/restic/restic/pull/4226 * Enhancement #4375: Add support for extended attributes on symlinks - Restic now supports extended attributes on symlinks when backing up, restoring, or - FUSE-mounting snapshots. This includes, for example, the `security.selinux` xattr on Linux - distributions that use SELinux. + Restic now supports extended attributes on symlinks when backing up, restoring, + or FUSE-mounting snapshots. This includes, for example, the `security.selinux` + xattr on Linux distributions that use SELinux. https://github.com/restic/restic/issues/4375 https://github.com/restic/restic/pull/4379 @@ -543,12 +639,12 @@ restic users. The changes are ordered by importance. * Bugfix #2260: Sanitize filenames printed by `backup` during processing - The `backup` command would previously not sanitize the filenames it printed during - processing, potentially causing newlines or terminal control characters to mangle the - status output or even change the state of a terminal. + The `backup` command would previously not sanitize the filenames it printed + during processing, potentially causing newlines or terminal control characters + to mangle the status output or even change the state of a terminal. - Filenames are now checked and quoted if they contain non-printable or non-Unicode - characters. + Filenames are now checked and quoted if they contain non-printable or + non-Unicode characters. https://github.com/restic/restic/issues/2260 https://github.com/restic/restic/issues/4191 @@ -557,44 +653,47 @@ restic users. The changes are ordered by importance. * Bugfix #4211: Make `dump` interpret `--host` and `--path` correctly A regression in restic 0.15.0 caused `dump` to confuse its `--host=` and - `--path=` options: it looked for snapshots with paths called `` from hosts - called ``. It now treats the options as intended. + `--path=` options: it looked for snapshots with paths called `` from + hosts called ``. It now treats the options as intended. https://github.com/restic/restic/issues/4211 https://github.com/restic/restic/pull/4212 * Bugfix #4239: Correct number of blocks reported in mount point - Restic mount points reported an incorrect number of 512-byte (POSIX standard) blocks for - files and links due to a rounding bug. In particular, empty files were reported as taking one - block instead of zero. + Restic mount points reported an incorrect number of 512-byte (POSIX standard) + blocks for files and links due to a rounding bug. In particular, empty files + were reported as taking one block instead of zero. - The rounding is now fixed: the number of blocks reported is the file size (or link target size) - divided by 512 and rounded up to a whole number. + The rounding is now fixed: the number of blocks reported is the file size (or + link target size) divided by 512 and rounded up to a whole number. https://github.com/restic/restic/issues/4239 https://github.com/restic/restic/pull/4240 * Bugfix #4253: Minimize risk of spurious filesystem loops with `mount` - When a backup contains a directory that has the same name as its parent, say `a/b/b`, and the GNU - `find` command was run on this backup in a restic mount, `find` would refuse to traverse the - lowest `b` directory, instead printing `File system loop detected`. This was due to the way the - restic mount command generates inode numbers for directories in the mount point. + When a backup contains a directory that has the same name as its parent, say + `a/b/b`, and the GNU `find` command was run on this backup in a restic mount, + `find` would refuse to traverse the lowest `b` directory, instead printing `File + system loop detected`. This was due to the way the restic mount command + generates inode numbers for directories in the mount point. - The rule for generating these inode numbers was changed in 0.15.0. It has now been changed again - to avoid this issue. A perfect rule does not exist, but the probability of this behavior - occurring is now extremely small. + The rule for generating these inode numbers was changed in 0.15.0. It has now + been changed again to avoid this issue. A perfect rule does not exist, but the + probability of this behavior occurring is now extremely small. - When it does occur, the mount point is not broken, and scripts that traverse the mount point - should work as long as they don't rely on inode numbers for detecting filesystem loops. + When it does occur, the mount point is not broken, and scripts that traverse the + mount point should work as long as they don't rely on inode numbers for + detecting filesystem loops. https://github.com/restic/restic/issues/4253 https://github.com/restic/restic/pull/4255 * Enhancement #4180: Add release binaries for riscv64 architecture on Linux - Builds for the `riscv64` architecture on Linux are now included in the release binaries. + Builds for the `riscv64` architecture on Linux are now included in the release + binaries. https://github.com/restic/restic/pull/4180 @@ -621,8 +720,8 @@ restic users. The changes are ordered by importance. * Bugfix #3750: Remove `b2_download_file_by_name: 404` warning from B2 backend - In some cases the B2 backend could print `b2_download_file_by_name: 404: : b2.b2err` - warnings. These are only debug messages and can be safely ignored. + In some cases the B2 backend could print `b2_download_file_by_name: 404: : + b2.b2err` warnings. These are only debug messages and can be safely ignored. Restic now uses an updated library for accessing B2, which removes the warning. @@ -632,19 +731,19 @@ restic users. The changes are ordered by importance. * Bugfix #4147: Make `prune --quiet` not print progress bar - A regression in restic 0.15.0 caused `prune --quiet` to show a progress bar while deciding how - to process each pack files. This has now been fixed. + A regression in restic 0.15.0 caused `prune --quiet` to show a progress bar + while deciding how to process each pack files. This has now been fixed. https://github.com/restic/restic/issues/4147 https://github.com/restic/restic/pull/4153 * Bugfix #4163: Make `self-update --output` work with new filename on Windows - Since restic 0.14.0 the `self-update` command did not work when a custom output filename was - specified via the `--output` option. This has now been fixed. + Since restic 0.14.0 the `self-update` command did not work when a custom output + filename was specified via the `--output` option. This has now been fixed. - As a workaround, either use an older restic version to run the self-update or create an empty - file with the output filename before updating e.g. using CMD: + As a workaround, either use an older restic version to run the self-update or + create an empty file with the output filename before updating e.g. using CMD: `type nul > new-file.exe` `restic self-update --output new-file.exe` @@ -653,24 +752,27 @@ restic users. The changes are ordered by importance. * Bugfix #4167: Add missing ETA in `backup` progress bar - A regression in restic 0.15.0 caused the ETA to be missing from the progress bar displayed by the - `backup` command. This has now been fixed. + A regression in restic 0.15.0 caused the ETA to be missing from the progress bar + displayed by the `backup` command. This has now been fixed. https://github.com/restic/restic/pull/4167 * Enhancement #4143: Ignore empty lock files - With restic 0.15.0 the checks for stale locks became much stricter than before. In particular, - empty or unreadable locks were no longer silently ignored. This made restic to complain with - `Load(, 0, 0) returned error, retrying after 552.330144ms: - load(): invalid data returned` and fail in the end. + With restic 0.15.0 the checks for stale locks became much stricter than before. + In particular, empty or unreadable locks were no longer silently ignored. This + made restic to complain with `Load(, 0, 0) returned error, + retrying after 552.330144ms: load(): invalid data returned` and + fail in the end. - The error message is now clarified and the implementation changed to ignore empty lock files - which are sometimes created as the result of a failed uploads on some backends. + The error message is now clarified and the implementation changed to ignore + empty lock files which are sometimes created as the result of a failed uploads + on some backends. - Please note that unreadable lock files still have to cleaned up manually. To do so, you can run - `restic unlock --remove-all` which removes all existing lock files. But first make sure that - no other restic process is currently using the repository. + Please note that unreadable lock files still have to cleaned up manually. To do + so, you can run `restic unlock --remove-all` which removes all existing lock + files. But first make sure that no other restic process is currently using the + repository. https://github.com/restic/restic/issues/4143 https://github.com/restic/restic/pull/4152 @@ -726,63 +828,65 @@ restic users. The changes are ordered by importance. * Bugfix #2015: Make `mount` return exit code 0 after receiving Ctrl-C / SIGINT - To stop the `mount` command, a user has to press Ctrl-C or send a SIGINT signal to restic. This - used to cause restic to exit with a non-zero exit code. + To stop the `mount` command, a user has to press Ctrl-C or send a SIGINT signal + to restic. This used to cause restic to exit with a non-zero exit code. - The exit code has now been changed to zero as the above is the expected way to stop the `mount` - command and should therefore be considered successful. + The exit code has now been changed to zero as the above is the expected way to + stop the `mount` command and should therefore be considered successful. https://github.com/restic/restic/issues/2015 https://github.com/restic/restic/pull/3894 * Bugfix #2578: Make `restore` replace existing symlinks - When restoring a symlink, restic used to report an error if the target path already existed. - This has now been fixed such that the potentially existing target path is first removed before - the symlink is restored. + When restoring a symlink, restic used to report an error if the target path + already existed. This has now been fixed such that the potentially existing + target path is first removed before the symlink is restored. https://github.com/restic/restic/issues/2578 https://github.com/restic/restic/pull/3780 * Bugfix #2591: Don't read password from stdin for `backup --stdin` - The `backup` command when used with `--stdin` previously tried to read first the password, - then the data to be backed up from standard input. This meant it would often confuse part of the - data for the password. + The `backup` command when used with `--stdin` previously tried to read first the + password, then the data to be backed up from standard input. This meant it would + often confuse part of the data for the password. - From now on, it will instead exit with the message `Fatal: cannot read both password and data - from stdin` unless the password is passed in some other way (such as - `--restic-password-file`, `RESTIC_PASSWORD`, etc). + From now on, it will instead exit with the message `Fatal: cannot read both + password and data from stdin` unless the password is passed in some other way + (such as `--restic-password-file`, `RESTIC_PASSWORD`, etc). - To enter the password interactively a password command has to be used. For example on Linux, - `mysqldump somedatabase | restic backup --stdin --password-command='sh -c - "systemd-ask-password < /dev/tty"'` securely reads the password from the terminal. + To enter the password interactively a password command has to be used. For + example on Linux, `mysqldump somedatabase | restic backup --stdin + --password-command='sh -c "systemd-ask-password < /dev/tty"'` securely reads the + password from the terminal. https://github.com/restic/restic/issues/2591 https://github.com/restic/restic/pull/4011 * Bugfix #3161: Delete files on Backblaze B2 more reliably - Restic used to only delete the latest version of files stored in B2. In most cases this worked - well as there was only a single version of the file. However, due to retries while uploading it is - possible for multiple file versions to be stored at B2. This could lead to various problems for - files that should have been deleted but still existed. + Restic used to only delete the latest version of files stored in B2. In most + cases this worked well as there was only a single version of the file. However, + due to retries while uploading it is possible for multiple file versions to be + stored at B2. This could lead to various problems for files that should have + been deleted but still existed. - The implementation has now been changed to delete all versions of files, which doubles the - amount of Class B transactions necessary to delete files, but assures that no file versions are - left behind. + The implementation has now been changed to delete all versions of files, which + doubles the amount of Class B transactions necessary to delete files, but + assures that no file versions are left behind. https://github.com/restic/restic/issues/3161 https://github.com/restic/restic/pull/3885 * Bugfix #3336: Make SFTP backend report no space left on device - Backing up to an SFTP backend would spew repeated SSH_FX_FAILURE messages when the remote disk - was full. Restic now reports "sftp: no space left on device" and exits immediately when it - detects this condition. + Backing up to an SFTP backend would spew repeated SSH_FX_FAILURE messages when + the remote disk was full. Restic now reports "sftp: no space left on device" and + exits immediately when it detects this condition. - A fix for this issue was implemented in restic 0.12.1, but unfortunately the fix itself - contained a bug that prevented it from taking effect. + A fix for this issue was implemented in restic 0.12.1, but unfortunately the fix + itself contained a bug that prevented it from taking effect. https://github.com/restic/restic/issues/3336 https://github.com/restic/restic/pull/3345 @@ -790,9 +894,10 @@ restic users. The changes are ordered by importance. * Bugfix #3567: Improve handling of interrupted syscalls in `mount` command - Accessing restic's FUSE mount could result in "input/output" errors when using programs in - which syscalls can be interrupted. This is for example the case for Go programs. This has now - been fixed by improved error handling of interrupted syscalls. + Accessing restic's FUSE mount could result in "input/output" errors when using + programs in which syscalls can be interrupted. This is for example the case for + Go programs. This has now been fixed by improved error handling of interrupted + syscalls. https://github.com/restic/restic/issues/3567 https://github.com/restic/restic/issues/3694 @@ -800,50 +905,53 @@ restic users. The changes are ordered by importance. * Bugfix #3897: Fix stuck `copy` command when `-o .connections=1` - When running the `copy` command with `-o .connections=1` the command would be - infinitely stuck. This has now been fixed. + When running the `copy` command with `-o .connections=1` the command + would be infinitely stuck. This has now been fixed. https://github.com/restic/restic/issues/3897 https://github.com/restic/restic/pull/3898 * Bugfix #3918: Correct prune statistics for partially compressed repositories - In a partially compressed repository, one data blob can exist both in an uncompressed and a - compressed version. This caused the `prune` statistics to become inaccurate and e.g. report a - too high value for the unused size, such as "unused size after prune: 16777215.991 TiB". This - has now been fixed. + In a partially compressed repository, one data blob can exist both in an + uncompressed and a compressed version. This caused the `prune` statistics to + become inaccurate and e.g. report a too high value for the unused size, such as + "unused size after prune: 16777215.991 TiB". This has now been fixed. https://github.com/restic/restic/issues/3918 https://github.com/restic/restic/pull/3980 * Bugfix #3951: Make `ls` return exit code 1 if snapshot cannot be loaded - The `ls` command used to show a warning and return exit code 0 when failing to load a snapshot. - This has now been fixed such that it instead returns exit code 1 (still showing a warning). + The `ls` command used to show a warning and return exit code 0 when failing to + load a snapshot. This has now been fixed such that it instead returns exit code + 1 (still showing a warning). https://github.com/restic/restic/pull/3951 * Bugfix #4003: Make `backup` no longer hang on Solaris when seeing a FIFO file - The `backup` command used to hang on Solaris whenever it encountered a FIFO file (named pipe), - due to a bug in the handling of extended attributes. This bug has now been fixed. + The `backup` command used to hang on Solaris whenever it encountered a FIFO file + (named pipe), due to a bug in the handling of extended attributes. This bug has + now been fixed. https://github.com/restic/restic/issues/4003 https://github.com/restic/restic/pull/4053 * Bugfix #4016: Support ExFAT-formatted local backends on macOS Ventura - ExFAT-formatted disks could not be used as local backends starting from macOS Ventura. Restic - commands would fail with an "inappropriate ioctl for device" error. This has now been fixed. + ExFAT-formatted disks could not be used as local backends starting from macOS + Ventura. Restic commands would fail with an "inappropriate ioctl for device" + error. This has now been fixed. https://github.com/restic/restic/issues/4016 https://github.com/restic/restic/pull/4021 * Bugfix #4085: Make `init` ignore "Access Denied" errors when creating S3 buckets - In restic 0.9.0 through 0.13.0, the `init` command ignored some permission errors from S3 - backends when trying to check for bucket existence, so that manually created buckets with - custom permissions could be used for backups. + In restic 0.9.0 through 0.13.0, the `init` command ignored some permission + errors from S3 backends when trying to check for bucket existence, so that + manually created buckets with custom permissions could be used for backups. This feature became broken in 0.14.0, but has now been restored again. @@ -852,20 +960,21 @@ restic users. The changes are ordered by importance. * Bugfix #4100: Make `self-update` enabled by default only in release builds - The `self-update` command was previously included by default in all builds of restic as - opposed to only in official release builds, even if the `selfupdate` tag was not explicitly - enabled when building. + The `self-update` command was previously included by default in all builds of + restic as opposed to only in official release builds, even if the `selfupdate` + tag was not explicitly enabled when building. - This has now been corrected, and the `self-update` command is only available if restic was - built with `-tags selfupdate` (as done for official release builds by `build.go`). + This has now been corrected, and the `self-update` command is only available if + restic was built with `-tags selfupdate` (as done for official release builds by + `build.go`). https://github.com/restic/restic/pull/4100 * Bugfix #4103: Don't generate negative UIDs and GIDs in tar files from `dump` - When using a 32-bit build of restic, the `dump` command could in some cases create tar files - containing negative UIDs and GIDs, which cannot be read by GNU tar. This corner case especially - applies to backups from stdin on Windows. + When using a 32-bit build of restic, the `dump` command could in some cases + create tar files containing negative UIDs and GIDs, which cannot be read by GNU + tar. This corner case especially applies to backups from stdin on Windows. This is now fixed such that `dump` creates valid tar files in these cases too. @@ -874,48 +983,50 @@ restic users. The changes are ordered by importance. * Change #2724: Include full snapshot ID in JSON output of `backup` - We have changed the JSON output of the backup command to include the full snapshot ID instead of - just a shortened version, as the latter can be ambiguous in some rare cases. To derive the short - ID, please truncate the full ID down to eight characters. + We have changed the JSON output of the backup command to include the full + snapshot ID instead of just a shortened version, as the latter can be ambiguous + in some rare cases. To derive the short ID, please truncate the full ID down to + eight characters. https://github.com/restic/restic/issues/2724 https://github.com/restic/restic/pull/3993 * Change #3929: Make `unlock` display message only when locks were actually removed - The `unlock` command used to print the "successfully removed locks" message whenever it was - run, regardless of lock files having being removed or not. + The `unlock` command used to print the "successfully removed locks" message + whenever it was run, regardless of lock files having being removed or not. - This has now been changed such that it only prints the message if any lock files were actually - removed. In addition, it also reports the number of removed lock files. + This has now been changed such that it only prints the message if any lock files + were actually removed. In addition, it also reports the number of removed lock + files. https://github.com/restic/restic/issues/3929 https://github.com/restic/restic/pull/3935 * Change #4033: Don't print skipped snapshots by default in `copy` command - The `copy` command used to print each snapshot that was skipped because it already existed in - the target repository. The amount of this output could practically bury the list of snapshots - that were actually copied. + The `copy` command used to print each snapshot that was skipped because it + already existed in the target repository. The amount of this output could + practically bury the list of snapshots that were actually copied. - From now on, the skipped snapshots are by default not printed at all, but this can be re-enabled - by increasing the verbosity level of the command. + From now on, the skipped snapshots are by default not printed at all, but this + can be re-enabled by increasing the verbosity level of the command. https://github.com/restic/restic/issues/4033 https://github.com/restic/restic/pull/4066 * Change #4041: Update dependencies and require Go 1.18 or newer - Most dependencies have been updated. Since some libraries require newer language features, - support for Go 1.15-1.17 has been dropped, which means that restic now requires at least Go 1.18 - to build. + Most dependencies have been updated. Since some libraries require newer language + features, support for Go 1.15-1.17 has been dropped, which means that restic now + requires at least Go 1.18 to build. https://github.com/restic/restic/pull/4041 * Enhancement #14: Implement `rewrite` command - Restic now has a `rewrite` command which allows to rewrite existing snapshots to remove - unwanted files. + Restic now has a `rewrite` command which allows to rewrite existing snapshots to + remove unwanted files. https://github.com/restic/restic/issues/14 https://github.com/restic/restic/pull/2731 @@ -923,15 +1034,15 @@ restic users. The changes are ordered by importance. * Enhancement #79: Restore files with long runs of zeros as sparse files - When using `restore --sparse`, the restorer may now write files containing long runs of zeros - as sparse files (also called files with holes), where the zeros are not actually written to - disk. + When using `restore --sparse`, the restorer may now write files containing long + runs of zeros as sparse files (also called files with holes), where the zeros + are not actually written to disk. - How much space is saved by writing sparse files depends on the operating system, file system and - the distribution of zeros in the file. + How much space is saved by writing sparse files depends on the operating system, + file system and the distribution of zeros in the file. - During backup restic still reads the whole file including sparse regions, but with optimized - processing speed of sparse regions. + During backup restic still reads the whole file including sparse regions, but + with optimized processing speed of sparse regions. https://github.com/restic/restic/issues/79 https://github.com/restic/restic/issues/3903 @@ -941,9 +1052,9 @@ restic users. The changes are ordered by importance. * Enhancement #1078: Support restoring symbolic links on Windows - The `restore` command now supports restoring symbolic links on Windows. Because of Windows - specific restrictions this is only possible when running restic with the - `SeCreateSymbolicLinkPrivilege` privilege or as an administrator. + The `restore` command now supports restoring symbolic links on Windows. Because + of Windows specific restrictions this is only possible when running restic with + the `SeCreateSymbolicLinkPrivilege` privilege or as an administrator. https://github.com/restic/restic/issues/1078 https://github.com/restic/restic/issues/2699 @@ -951,14 +1062,14 @@ restic users. The changes are ordered by importance. * Enhancement #1734: Inform about successful retries after errors - When a recoverable error is encountered, restic shows a warning message saying that it's - retrying, e.g.: + When a recoverable error is encountered, restic shows a warning message saying + that it's retrying, e.g.: `Save() returned error, retrying after 357.131936ms: ...` - This message can be confusing in that it never clearly states whether the retry is successful or - not. This has now been fixed such that restic follows up with a message confirming a successful - retry, e.g.: + This message can be confusing in that it never clearly states whether the retry + is successful or not. This has now been fixed such that restic follows up with a + message confirming a successful retry, e.g.: `Save() operation successful after 1 retries` @@ -967,12 +1078,12 @@ restic users. The changes are ordered by importance. * Enhancement #1866: Improve handling of directories with duplicate entries - If for some reason a directory contains a duplicate entry, the `backup` command would - previously fail with a `node "path/to/file" already present` or `nodes are not ordered got - "path/to/file", last "path/to/file"` error. + If for some reason a directory contains a duplicate entry, the `backup` command + would previously fail with a `node "path/to/file" already present` or `nodes are + not ordered got "path/to/file", last "path/to/file"` error. - The error handling has been improved to only report a warning in this case. Make sure to check - that the filesystem in question is not damaged if you see this! + The error handling has been improved to only report a warning in this case. Make + sure to check that the filesystem in question is not damaged if you see this! https://github.com/restic/restic/issues/1866 https://github.com/restic/restic/issues/3937 @@ -980,29 +1091,31 @@ restic users. The changes are ordered by importance. * Enhancement #2134: Support B2 API keys restricted to hiding but not deleting files - When the B2 backend does not have the necessary permissions to permanently delete files, it now - automatically falls back to hiding files. This allows using restic with an application key - which is not allowed to delete files. This can prevent an attacker from deleting backups with - such an API key. + When the B2 backend does not have the necessary permissions to permanently + delete files, it now automatically falls back to hiding files. This allows using + restic with an application key which is not allowed to delete files. This can + prevent an attacker from deleting backups with such an API key. - To use this feature create an application key without the `deleteFiles` capability. It is - recommended to restrict the key to just one bucket. For example using the `b2` command line - tool: + To use this feature create an application key without the `deleteFiles` + capability. It is recommended to restrict the key to just one bucket. For + example using the `b2` command line tool: `b2 create-key --bucket listBuckets,readFiles,writeFiles,listFiles` - Alternatively, you can use the S3 backend to access B2, as described in the documentation. In - this mode, files are also only hidden instead of being deleted permanently. + Alternatively, you can use the S3 backend to access B2, as described in the + documentation. In this mode, files are also only hidden instead of being deleted + permanently. https://github.com/restic/restic/issues/2134 https://github.com/restic/restic/pull/2398 * Enhancement #2152: Make `init` open only one connection for the SFTP backend - The `init` command using the SFTP backend used to connect twice to the repository. This could be - inconvenient if the user must enter a password, or cause `init` to fail if the server does not - correctly close the first SFTP connection. + The `init` command using the SFTP backend used to connect twice to the + repository. This could be inconvenient if the user must enter a password, or + cause `init` to fail if the server does not correctly close the first SFTP + connection. This has now been fixed by reusing the first/initial SFTP connection opened. @@ -1011,40 +1124,44 @@ restic users. The changes are ordered by importance. * Enhancement #2533: Handle cache corruption on disk and in downloads - In rare situations, like for example after a system crash, the data stored in the cache might be - corrupted. This could cause restic to fail and required manually deleting the cache. + In rare situations, like for example after a system crash, the data stored in + the cache might be corrupted. This could cause restic to fail and required + manually deleting the cache. - Restic now automatically removes broken data from the cache, allowing it to recover from such a - situation without user intervention. In addition, restic retries downloads which return - corrupt data in order to also handle temporary download problems. + Restic now automatically removes broken data from the cache, allowing it to + recover from such a situation without user intervention. In addition, restic + retries downloads which return corrupt data in order to also handle temporary + download problems. https://github.com/restic/restic/issues/2533 https://github.com/restic/restic/pull/3521 * Enhancement #2715: Stricter repository lock handling - Previously, restic commands kept running even if they failed to refresh their locks in time. - This could be a problem e.g. in case the client system running a backup entered the standby power - mode while the backup was still in progress (which would prevent the client from refreshing its - lock), and after a short delay another host successfully runs `unlock` and `prune` on the - repository, which would remove all data added by the in-progress backup. If the backup client - later continues its backup, even though its lock had expired in the meantime, this would lead to - an incomplete snapshot. + Previously, restic commands kept running even if they failed to refresh their + locks in time. This could be a problem e.g. in case the client system running a + backup entered the standby power mode while the backup was still in progress + (which would prevent the client from refreshing its lock), and after a short + delay another host successfully runs `unlock` and `prune` on the repository, + which would remove all data added by the in-progress backup. If the backup + client later continues its backup, even though its lock had expired in the + meantime, this would lead to an incomplete snapshot. - To address this, lock handling is now much stricter. Commands requiring a lock are canceled if - the lock is not refreshed successfully in time. In addition, if a lock file is not readable - restic will not allow starting a command. It may be necessary to remove invalid lock files - manually or use `unlock --remove-all`. Please make sure that no other restic processes are - running concurrently before doing this, however. + To address this, lock handling is now much stricter. Commands requiring a lock + are canceled if the lock is not refreshed successfully in time. In addition, if + a lock file is not readable restic will not allow starting a command. It may be + necessary to remove invalid lock files manually or use `unlock --remove-all`. + Please make sure that no other restic processes are running concurrently before + doing this, however. https://github.com/restic/restic/issues/2715 https://github.com/restic/restic/pull/3569 * Enhancement #2750: Make backup file read concurrency configurable - The `backup` command now supports a `--read-concurrency` option which allows tuning restic - for very fast storage like NVMe disks by controlling the number of concurrent file reads during - the backup process. + The `backup` command now supports a `--read-concurrency` option which allows + tuning restic for very fast storage like NVMe disks by controlling the number of + concurrent file reads during the backup process. https://github.com/restic/restic/pull/2750 @@ -1059,75 +1176,78 @@ restic users. The changes are ordered by importance. * Enhancement #3096: Make `mount` command support macOS using macFUSE 4.x - Restic now uses a different FUSE library for mounting snapshots and making them available as a - FUSE filesystem using the `mount` command. This adds support for macFUSE 4.x which can be used - to make this work on recent macOS versions. + Restic now uses a different FUSE library for mounting snapshots and making them + available as a FUSE filesystem using the `mount` command. This adds support for + macFUSE 4.x which can be used to make this work on recent macOS versions. https://github.com/restic/restic/issues/3096 https://github.com/restic/restic/pull/4024 * Enhancement #3124: Support JSON output for the `init` command - The `init` command used to ignore the `--json` option, but now outputs a JSON message if the - repository was created successfully. + The `init` command used to ignore the `--json` option, but now outputs a JSON + message if the repository was created successfully. https://github.com/restic/restic/issues/3124 https://github.com/restic/restic/pull/3132 * Enhancement #3899: Optimize prune memory usage - The `prune` command needs large amounts of memory in order to determine what to keep and what to - remove. This is now optimized to use up to 30% less memory. + The `prune` command needs large amounts of memory in order to determine what to + keep and what to remove. This is now optimized to use up to 30% less memory. https://github.com/restic/restic/pull/3899 * Enhancement #3905: Improve speed of parent snapshot detection in `backup` command - Backing up a large number of files using `--files-from-verbatim` or `--files-from-raw` - options could require a long time to find the parent snapshot. This has been improved. + Backing up a large number of files using `--files-from-verbatim` or + `--files-from-raw` options could require a long time to find the parent + snapshot. This has been improved. https://github.com/restic/restic/pull/3905 * Enhancement #3915: Add compression statistics to the `stats` command - When executed with `--mode raw-data` on a repository that supports compression, the `stats` - command now calculates and displays, for the selected repository or snapshots: the - uncompressed size of the data; the compression progress (percentage of data that has been - compressed); the compression ratio of the compressed data; the total space saving. + When executed with `--mode raw-data` on a repository that supports compression, + the `stats` command now calculates and displays, for the selected repository or + snapshots: the uncompressed size of the data; the compression progress + (percentage of data that has been compressed); the compression ratio of the + compressed data; the total space saving. - It also takes into account both the compressed and uncompressed data if the repository is only - partially compressed. + It also takes into account both the compressed and uncompressed data if the + repository is only partially compressed. https://github.com/restic/restic/pull/3915 * Enhancement #3925: Provide command completion for PowerShell - Restic already provided generation of completion files for bash, fish and zsh. Now powershell - is supported, too. + Restic already provided generation of completion files for bash, fish and zsh. + Now powershell is supported, too. https://github.com/restic/restic/pull/3925/files * Enhancement #3931: Allow `backup` file tree scanner to be disabled - The `backup` command walks the file tree in a separate scanner process to find the total size and - file/directory count, and uses this to provide an ETA. This can slow down backups, especially - of network filesystems. + The `backup` command walks the file tree in a separate scanner process to find + the total size and file/directory count, and uses this to provide an ETA. This + can slow down backups, especially of network filesystems. - The command now has a new option `--no-scan` which can be used to disable this scanning in order - to speed up backups when needed. + The command now has a new option `--no-scan` which can be used to disable this + scanning in order to speed up backups when needed. https://github.com/restic/restic/pull/3931 * Enhancement #3932: Improve handling of ErrDot errors in rclone and sftp backends - Since Go 1.19, restic can no longer implicitly run relative executables which are found in the - current directory (e.g. `rclone` if found in `.`). This is a security feature of Go to prevent - against running unintended and possibly harmful executables. + Since Go 1.19, restic can no longer implicitly run relative executables which + are found in the current directory (e.g. `rclone` if found in `.`). This is a + security feature of Go to prevent against running unintended and possibly + harmful executables. - The error message for this was just "cannot run executable found relative to current - directory". This has now been improved to yield a more specific error message, informing the - user how to explicitly allow running the executable using the `-o rclone.program` and `-o - sftp.command` extended options with `./`. + The error message for this was just "cannot run executable found relative to + current directory". This has now been improved to yield a more specific error + message, informing the user how to explicitly allow running the executable using + the `-o rclone.program` and `-o sftp.command` extended options with `./`. https://github.com/restic/restic/issues/3932 https://pkg.go.dev/os/exec#hdr-Executables_in_the_current_directory @@ -1135,20 +1255,21 @@ restic users. The changes are ordered by importance. * Enhancement #3943: Ignore additional/unknown files in repository - If a restic repository had additional files in it (not created by restic), commands like `find` - and `restore` could become confused and fail with an `multiple IDs with prefix "12345678" - found` error. These commands now ignore such additional files. + If a restic repository had additional files in it (not created by restic), + commands like `find` and `restore` could become confused and fail with an + `multiple IDs with prefix "12345678" found` error. These commands now ignore + such additional files. https://github.com/restic/restic/pull/3943 https://forum.restic.net/t/which-protocol-should-i-choose-for-remote-linux-backups/5446/17 * Enhancement #3955: Improve `backup` performance for small files - When backing up small files restic was slower than it could be. In particular this affected - backups using maximum compression. + When backing up small files restic was slower than it could be. In particular + this affected backups using maximum compression. - This has been fixed by reworking the internal parallelism of the backup command, making it back - up small files around two times faster. + This has been fixed by reworking the internal parallelism of the backup command, + making it back up small files around two times faster. https://github.com/restic/restic/pull/3955 @@ -1197,22 +1318,23 @@ restic users. The changes are ordered by importance. * Bugfix #2248: Support `self-update` on Windows - Restic `self-update` would fail in situations where the operating system locks running - binaries, including Windows. The new behavior works around this by renaming the running file - and swapping the updated file in place. + Restic `self-update` would fail in situations where the operating system locks + running binaries, including Windows. The new behavior works around this by + renaming the running file and swapping the updated file in place. https://github.com/restic/restic/issues/2248 https://github.com/restic/restic/pull/3675 * Bugfix #3428: List snapshots in backend at most once to resolve snapshot IDs - Many commands support specifying a list of snapshot IDs which are then used to determine the - snapshots to be processed by the command. To resolve snapshot IDs or `latest`, and check that - these exist, restic previously listed all snapshots stored in the repository. Depending on - the backend this could be a slow and/or expensive operation. + Many commands support specifying a list of snapshot IDs which are then used to + determine the snapshots to be processed by the command. To resolve snapshot IDs + or `latest`, and check that these exist, restic previously listed all snapshots + stored in the repository. Depending on the backend this could be a slow and/or + expensive operation. - Restic now lists the snapshots only once and remembers the result in order to resolve all - further snapshot IDs swiftly. + Restic now lists the snapshots only once and remembers the result in order to + resolve all further snapshot IDs swiftly. https://github.com/restic/restic/issues/3428 https://github.com/restic/restic/pull/3570 @@ -1220,27 +1342,28 @@ restic users. The changes are ordered by importance. * Bugfix #3432: Fix rare 'not found in repository' error for `copy` command - In rare cases `copy` (and other commands) would report that `LoadTree(...)` returned an `id - [...] not found in repository` error. This could be caused by a backup or copy command running - concurrently. The error was only temporary; running the failed restic command a second time as - a workaround did resolve the error. + In rare cases `copy` (and other commands) would report that `LoadTree(...)` + returned an `id [...] not found in repository` error. This could be caused by a + backup or copy command running concurrently. The error was only temporary; + running the failed restic command a second time as a workaround did resolve the + error. - This issue has now been fixed by correcting the order in which restic reads data from the - repository. It is now guaranteed that restic only loads snapshots for which all necessary data - is already available. + This issue has now been fixed by correcting the order in which restic reads data + from the repository. It is now guaranteed that restic only loads snapshots for + which all necessary data is already available. https://github.com/restic/restic/issues/3432 https://github.com/restic/restic/pull/3570 * Bugfix #3681: Fix rclone (shimmed by Scoop) and sftp not working on Windows - In #3602 a fix was introduced to address the problem of `rclone` prematurely exiting when - Ctrl+C is pressed on Windows. The solution was to create the subprocess with its console - detached from the restic console. + In #3602 a fix was introduced to address the problem of `rclone` prematurely + exiting when Ctrl+C is pressed on Windows. The solution was to create the + subprocess with its console detached from the restic console. - However, this solution failed when using `rclone` installed by Scoop or using `sftp` with a - passphrase-protected private key. We've now fixed this by using a different approach to - prevent Ctrl-C from passing down too early. + However, this solution failed when using `rclone` installed by Scoop or using + `sftp` with a passphrase-protected private key. We've now fixed this by using a + different approach to prevent Ctrl-C from passing down too early. https://github.com/restic/restic/issues/3681 https://github.com/restic/restic/issues/3692 @@ -1248,28 +1371,28 @@ restic users. The changes are ordered by importance. * Bugfix #3685: The `diff` command incorrectly listed some files as added - There was a bug in the `diff` command, causing it to always show files in a removed directory as - added. This has now been fixed. + There was a bug in the `diff` command, causing it to always show files in a + removed directory as added. This has now been fixed. https://github.com/restic/restic/issues/3685 https://github.com/restic/restic/pull/3686 * Bugfix #3716: Print "wrong password" to stderr instead of stdout - If an invalid password was entered, the error message was printed on stdout and not on stderr as - intended. This has now been fixed. + If an invalid password was entered, the error message was printed on stdout and + not on stderr as intended. This has now been fixed. https://github.com/restic/restic/pull/3716 https://forum.restic.net/t/4965 * Bugfix #3720: Directory sync errors for repositories accessed via SMB - On Linux and macOS, accessing a repository via a SMB/CIFS mount resulted in restic failing to - save the lock file, yielding the following errors: + On Linux and macOS, accessing a repository via a SMB/CIFS mount resulted in + restic failing to save the lock file, yielding the following errors: - Save() returned error, retrying after 552.330144ms: sync /repo/locks: - no such file or directory Save() returned error, retrying after - 552.330144ms: sync /repo/locks: invalid argument + Save() returned error, retrying after 552.330144ms: sync + /repo/locks: no such file or directory Save() returned error, + retrying after 552.330144ms: sync /repo/locks: invalid argument This has now been fixed by ignoring the relevant error codes. @@ -1279,22 +1402,23 @@ restic users. The changes are ordered by importance. * Bugfix #3736: The `stats` command miscalculated restore size for multiple snapshots - Since restic 0.10.0 the restore size calculated by the `stats` command for multiple snapshots - was too low. The hardlink detection was accidentally applied across multiple snapshots and - thus ignored many files. This has now been fixed. + Since restic 0.10.0 the restore size calculated by the `stats` command for + multiple snapshots was too low. The hardlink detection was accidentally applied + across multiple snapshots and thus ignored many files. This has now been fixed. https://github.com/restic/restic/issues/3736 https://github.com/restic/restic/pull/3740 * Bugfix #3772: Correctly rebuild index for legacy repositories - After running `rebuild-index` on a legacy repository containing mixed pack files (that is, - pack files which store both metadata and file data), `check` printed warnings like `pack - 12345678 contained in several indexes: ...`. This warning was not critical, but has now - nonetheless been fixed by properly handling mixed pack files while rebuilding the index. + After running `rebuild-index` on a legacy repository containing mixed pack files + (that is, pack files which store both metadata and file data), `check` printed + warnings like `pack 12345678 contained in several indexes: ...`. This warning + was not critical, but has now nonetheless been fixed by properly handling mixed + pack files while rebuilding the index. - Running `prune` for such legacy repositories will also fix the warning by reorganizing the - pack files which caused it. + Running `prune` for such legacy repositories will also fix the warning by + reorganizing the pack files which caused it. https://github.com/restic/restic/pull/3772 https://github.com/restic/restic/pull/3884 @@ -1302,18 +1426,20 @@ restic users. The changes are ordered by importance. * Bugfix #3776: Limit number of key files tested while opening a repository - Previously, restic tested the password against every key in the repository when opening a - repository. The more keys there were in the repository, the slower this operation became. + Previously, restic tested the password against every key in the repository when + opening a repository. The more keys there were in the repository, the slower + this operation became. - Restic now tests the password against up to 20 key files in the repository. Alternatively, you - can use the `--key-hint=` option to specify a specific key file to use instead. + Restic now tests the password against up to 20 key files in the repository. + Alternatively, you can use the `--key-hint=` option to specify a + specific key file to use instead. https://github.com/restic/restic/pull/3776 * Bugfix #3861: Yield error on invalid policy to `forget` - The `forget` command previously silently ignored invalid/unsupported units in the duration - options, such as e.g. `--keep-within-daily 2w`. + The `forget` command previously silently ignored invalid/unsupported units in + the duration options, such as e.g. `--keep-within-daily 2w`. Specifying an invalid/unsupported duration unit now results in an error. @@ -1322,71 +1448,78 @@ restic users. The changes are ordered by importance. * Change #1842: Support debug log creation in release builds - Creating a debug log was only possible in debug builds which required users to manually build - restic. We changed the release builds to allow creating debug logs by simply setting the - environment variable `DEBUG_LOG=logname.log`. + Creating a debug log was only possible in debug builds which required users to + manually build restic. We changed the release builds to allow creating debug + logs by simply setting the environment variable `DEBUG_LOG=logname.log`. https://github.com/restic/restic/issues/1842 https://github.com/restic/restic/pull/3826 * Change #3295: Deprecate `check --check-unused` and add further checks - Since restic 0.12.0, it is expected to still have unused blobs after running `prune`. This made - the `--check-unused` option of the `check` command rather useless and tended to confuse - users. This option has been deprecated and is now ignored. + Since restic 0.12.0, it is expected to still have unused blobs after running + `prune`. This made the `--check-unused` option of the `check` command rather + useless and tended to confuse users. This option has been deprecated and is now + ignored. - The `check` command now also warns if a repository is using either the legacy S3 layout or mixed - pack files with both tree and data blobs. The latter is known to cause performance problems. + The `check` command now also warns if a repository is using either the legacy S3 + layout or mixed pack files with both tree and data blobs. The latter is known to + cause performance problems. https://github.com/restic/restic/issues/3295 https://github.com/restic/restic/pull/3730 * Change #3680: Update dependencies and require Go 1.15 or newer - We've updated most dependencies. Since some libraries require newer language features we're - dropping support for Go 1.14, which means that restic now requires at least Go 1.15 to build. + We've updated most dependencies. Since some libraries require newer language + features we're dropping support for Go 1.14, which means that restic now + requires at least Go 1.15 to build. https://github.com/restic/restic/issues/3680 https://github.com/restic/restic/issues/3883 * Change #3742: Replace `--repo2` option used by `init`/`copy` with `--from-repo` - The `init` and `copy` commands can read data from another repository. However, confusingly - `--repo2` referred to the repository *from* which the `init` command copies parameters, but - for the `copy` command `--repo2` referred to the copy *destination*. + The `init` and `copy` commands can read data from another repository. However, + confusingly `--repo2` referred to the repository *from* which the `init` command + copies parameters, but for the `copy` command `--repo2` referred to the copy + *destination*. - We've introduced a new option, `--from-repo`, which always refers to the source repository - for both commands. The old parameter names have been deprecated but still work. To create a new - repository and copy all snapshots to it, the commands are now as follows: + We've introduced a new option, `--from-repo`, which always refers to the source + repository for both commands. The old parameter names have been deprecated but + still work. To create a new repository and copy all snapshots to it, the + commands are now as follows: - ``` restic -r /srv/restic-repo-copy init --from-repo /srv/restic-repo - --copy-chunker-params restic -r /srv/restic-repo-copy copy --from-repo - /srv/restic-repo ``` + ``` + restic -r /srv/restic-repo-copy init --from-repo /srv/restic-repo --copy-chunker-params + restic -r /srv/restic-repo-copy copy --from-repo /srv/restic-repo + ``` https://github.com/restic/restic/pull/3742 https://forum.restic.net/t/5017 * Enhancement #21: Add compression support - We've added compression support to the restic repository format. To create a repository using - the new format run `init --repository-version 2`. Please note that the repository cannot be - read by restic versions prior to 0.14.0. + We've added compression support to the restic repository format. To create a + repository using the new format run `init --repository-version 2`. Please note + that the repository cannot be read by restic versions prior to 0.14.0. - You can configure whether data is compressed with the option `--compression`. It can be set to - `auto` (the default, which will compress very fast), `max` (which will trade backup speed and - CPU usage for better compression), or `off` (which disables compression). Each setting is - only applied for the current run of restic and does *not* apply to future runs. The option can - also be set via the environment variable `RESTIC_COMPRESSION`. + You can configure whether data is compressed with the option `--compression`. It + can be set to `auto` (the default, which will compress very fast), `max` (which + will trade backup speed and CPU usage for better compression), or `off` (which + disables compression). Each setting is only applied for the current run of + restic and does *not* apply to future runs. The option can also be set via the + environment variable `RESTIC_COMPRESSION`. - To upgrade in place run `migrate upgrade_repo_v2` followed by `prune`. See the documentation - for more details. The migration checks the repository integrity and upgrades the repository - format, but will not change any data. Afterwards, prune will rewrite the metadata to make use of - compression. + To upgrade in place run `migrate upgrade_repo_v2` followed by `prune`. See the + documentation for more details. The migration checks the repository integrity + and upgrades the repository format, but will not change any data. Afterwards, + prune will rewrite the metadata to make use of compression. - As an alternative you can use the `copy` command to migrate snapshots; First create a new - repository using `init --repository-version 2 --copy-chunker-params --repo2 - path/to/old/repo`, and then use the `copy` command to copy all snapshots to the new - repository. + As an alternative you can use the `copy` command to migrate snapshots; First + create a new repository using `init --repository-version 2 --copy-chunker-params + --repo2 path/to/old/repo`, and then use the `copy` command to copy all snapshots + to the new repository. https://github.com/restic/restic/issues/21 https://github.com/restic/restic/issues/3779 @@ -1396,25 +1529,28 @@ restic users. The changes are ordered by importance. * Enhancement #1153: Support pruning even when the disk is full - When running out of disk space it was no longer possible to add or remove data from a repository. - To help with recovering from such a deadlock, the prune command now supports an - `--unsafe-recover-no-free-space` option to recover from these situations. Make sure to - read the documentation first! + When running out of disk space it was no longer possible to add or remove data + from a repository. To help with recovering from such a deadlock, the prune + command now supports an `--unsafe-recover-no-free-space` option to recover from + these situations. Make sure to read the documentation first! https://github.com/restic/restic/issues/1153 https://github.com/restic/restic/pull/3481 * Enhancement #2162: Adaptive IO concurrency based on backend connections - Many commands used hard-coded limits for the number of concurrent operations. This prevented - speed improvements by increasing the number of connections used by a backend. + Many commands used hard-coded limits for the number of concurrent operations. + This prevented speed improvements by increasing the number of connections used + by a backend. - These limits have now been replaced by using the configured number of backend connections - instead, which can be controlled using the `-o .connections=5` option. - Commands will then automatically scale their parallelism accordingly. + These limits have now been replaced by using the configured number of backend + connections instead, which can be controlled using the `-o + .connections=5` option. Commands will then automatically scale + their parallelism accordingly. - To limit the number of CPU cores used by restic, you can set the environment variable - `GOMAXPROCS` accordingly. For example to use a single CPU core, use `GOMAXPROCS=1`. + To limit the number of CPU cores used by restic, you can set the environment + variable `GOMAXPROCS` accordingly. For example to use a single CPU core, use + `GOMAXPROCS=1`. https://github.com/restic/restic/issues/2162 https://github.com/restic/restic/issues/1467 @@ -1422,45 +1558,47 @@ restic users. The changes are ordered by importance. * Enhancement #2291: Allow pack size customization - Restic now uses a target pack size of 16 MiB by default. This can be customized using the - `--pack-size size` option. Supported pack sizes range between 4 and 128 MiB. + Restic now uses a target pack size of 16 MiB by default. This can be customized + using the `--pack-size size` option. Supported pack sizes range between 4 and + 128 MiB. - It is possible to migrate an existing repository to _larger_ pack files using `prune - --repack-small`. This will rewrite every pack file which is significantly smaller than the - target size. + It is possible to migrate an existing repository to _larger_ pack files using + `prune --repack-small`. This will rewrite every pack file which is significantly + smaller than the target size. https://github.com/restic/restic/issues/2291 https://github.com/restic/restic/pull/3731 * Enhancement #2295: Allow use of SAS token to authenticate to Azure - Previously restic only supported AccountKeys to authenticate to Azure storage accounts, - which necessitates giving a significant amount of access. + Previously restic only supported AccountKeys to authenticate to Azure storage + accounts, which necessitates giving a significant amount of access. - We added support for Azure SAS tokens which are a more fine-grained and time-limited manner of - granting access. Set the `AZURE_ACCOUNT_NAME` and `AZURE_ACCOUNT_SAS` environment - variables to use a SAS token for authentication. Note that if `AZURE_ACCOUNT_KEY` is set, it - will take precedence. + We added support for Azure SAS tokens which are a more fine-grained and + time-limited manner of granting access. Set the `AZURE_ACCOUNT_NAME` and + `AZURE_ACCOUNT_SAS` environment variables to use a SAS token for authentication. + Note that if `AZURE_ACCOUNT_KEY` is set, it will take precedence. https://github.com/restic/restic/issues/2295 https://github.com/restic/restic/pull/3661 * Enhancement #2351: Use config file permissions to control file group access - Previously files in a local/SFTP repository would always end up with very restrictive access - permissions, allowing access only to the owner. This prevented a number of valid use-cases - involving groups and ACLs. + Previously files in a local/SFTP repository would always end up with very + restrictive access permissions, allowing access only to the owner. This + prevented a number of valid use-cases involving groups and ACLs. - We now use the permissions of the config file in the repository to decide whether group access - should be given to newly created repository files or not. We arrange for repository files to be - created group readable exactly when the repository config file is group readable. + We now use the permissions of the config file in the repository to decide + whether group access should be given to newly created repository files or not. + We arrange for repository files to be created group readable exactly when the + repository config file is group readable. - To opt-in to group readable repositories, a simple `chmod -R g+r` or equivalent on the config - file can be used. For repositories that should be writable by group members a tad more setup is - required, see the docs. + To opt-in to group readable repositories, a simple `chmod -R g+r` or equivalent + on the config file can be used. For repositories that should be writable by + group members a tad more setup is required, see the docs. - Posix ACLs can also be used now that the group permissions being forced to zero no longer masks - the effect of ACL entries. + Posix ACLs can also be used now that the group permissions being forced to zero + no longer masks the effect of ACL entries. https://github.com/restic/restic/issues/2351 https://github.com/restic/restic/pull/3419 @@ -1468,27 +1606,29 @@ restic users. The changes are ordered by importance. * Enhancement #2696: Improve backup speed with many small files - We have restructured the backup pipeline to continue reading files while all upload - connections are busy. This allows the backup to already prepare the next data file such that the - upload can continue as soon as a connection becomes available. This can especially improve the - backup performance for high latency backends. + We have restructured the backup pipeline to continue reading files while all + upload connections are busy. This allows the backup to already prepare the next + data file such that the upload can continue as soon as a connection becomes + available. This can especially improve the backup performance for high latency + backends. - The upload concurrency is now controlled using the `-o .connections=5` - option. + The upload concurrency is now controlled using the `-o + .connections=5` option. https://github.com/restic/restic/issues/2696 https://github.com/restic/restic/pull/3489 * Enhancement #2907: Make snapshot directory structure of `mount` command customizable - We've added the possibility to customize the snapshot directory structure of the `mount` - command using templates passed to the `--snapshot-template` option. The formatting of - snapshots' timestamps is now controlled using `--time-template` and supports - subdirectories to for example group snapshots by year. Please see `restic help mount` for - further details. + We've added the possibility to customize the snapshot directory structure of the + `mount` command using templates passed to the `--snapshot-template` option. The + formatting of snapshots' timestamps is now controlled using `--time-template` + and supports subdirectories to for example group snapshots by year. Please see + `restic help mount` for further details. - Characters in tag names which are not allowed in a filename are replaced by underscores `_`. For - example a tag `foo/bar` will result in a directory name of `foo_bar`. + Characters in tag names which are not allowed in a filename are replaced by + underscores `_`. For example a tag `foo/bar` will result in a directory name of + `foo_bar`. https://github.com/restic/restic/issues/2907 https://github.com/restic/restic/pull/2913 @@ -1496,8 +1636,9 @@ restic users. The changes are ordered by importance. * Enhancement #2923: Improve speed of `copy` command - The `copy` command could require a long time to copy snapshots for non-local backends. This has - been improved to provide a throughput comparable to the `restore` command. + The `copy` command could require a long time to copy snapshots for non-local + backends. This has been improved to provide a throughput comparable to the + `restore` command. Additionally, `copy` now displays a progress bar. @@ -1506,21 +1647,23 @@ restic users. The changes are ordered by importance. * Enhancement #3114: Optimize handling of duplicate blobs in `prune` - Restic `prune` always used to repack all data files containing duplicate blobs. This - effectively removed all duplicates during prune. However, as a consequence all these data - files were repacked even if the unused repository space threshold could be reached with less - work. + Restic `prune` always used to repack all data files containing duplicate blobs. + This effectively removed all duplicates during prune. However, as a consequence + all these data files were repacked even if the unused repository space threshold + could be reached with less work. - This is now changed and `prune` works nice and fast even when there are lots of duplicate blobs. + This is now changed and `prune` works nice and fast even when there are lots of + duplicate blobs. https://github.com/restic/restic/issues/3114 https://github.com/restic/restic/pull/3290 * Enhancement #3465: Improve handling of temporary files on Windows - In some cases restic failed to delete temporary files, causing the current command to fail. - This has now been fixed by ensuring that Windows automatically deletes the file. In addition, - temporary files are only written to disk when necessary, reducing disk writes. + In some cases restic failed to delete temporary files, causing the current + command to fail. This has now been fixed by ensuring that Windows automatically + deletes the file. In addition, temporary files are only written to disk when + necessary, reducing disk writes. https://github.com/restic/restic/issues/3465 https://github.com/restic/restic/issues/1551 @@ -1528,22 +1671,23 @@ restic users. The changes are ordered by importance. * Enhancement #3475: Allow limiting IO concurrency for local and SFTP backend - Restic did not support limiting the IO concurrency / number of connections for accessing - repositories stored using the local or SFTP backends. The number of connections is now limited - as for other backends, and can be configured via the `-o local.connections=2` and `-o - sftp.connections=5` options. This ensures that restic does not overwhelm the backend with - concurrent IO operations. + Restic did not support limiting the IO concurrency / number of connections for + accessing repositories stored using the local or SFTP backends. The number of + connections is now limited as for other backends, and can be configured via the + `-o local.connections=2` and `-o sftp.connections=5` options. This ensures that + restic does not overwhelm the backend with concurrent IO operations. https://github.com/restic/restic/pull/3475 * Enhancement #3484: Stream data in `check` and `prune` commands - The commands `check --read-data` and `prune` previously downloaded data files into - temporary files which could end up being written to disk. This could cause a large amount of data - being written to disk. + The commands `check --read-data` and `prune` previously downloaded data files + into temporary files which could end up being written to disk. This could cause + a large amount of data being written to disk. - The pack files are now instead streamed, which removes the need for temporary files. Please - note that *uploads* during `backup` and `prune` still require temporary files. + The pack files are now instead streamed, which removes the need for temporary + files. Please note that *uploads* during `backup` and `prune` still require + temporary files. https://github.com/restic/restic/issues/3710 https://github.com/restic/restic/pull/3484 @@ -1552,19 +1696,19 @@ restic users. The changes are ordered by importance. * Enhancement #3709: Validate exclude patterns before backing up Exclude patterns provided via `--exclude`, `--iexclude`, `--exclude-file` or - `--iexclude-file` previously weren't validated. As a consequence, invalid patterns - resulted in files that were meant to be excluded being backed up. + `--iexclude-file` previously weren't validated. As a consequence, invalid + patterns resulted in files that were meant to be excluded being backed up. - Restic now validates all patterns before running the backup and aborts with a fatal error if an - invalid pattern is detected. + Restic now validates all patterns before running the backup and aborts with a + fatal error if an invalid pattern is detected. https://github.com/restic/restic/issues/3709 https://github.com/restic/restic/pull/3734 * Enhancement #3729: Display full IDs in `check` warnings - When running commands to inspect or repair a damaged repository, it is often necessary to - supply the full IDs of objects stored in the repository. + When running commands to inspect or repair a damaged repository, it is often + necessary to supply the full IDs of objects stored in the repository. The output of `check` now includes full IDs instead of their shortened variant. @@ -1572,28 +1716,29 @@ restic users. The changes are ordered by importance. * Enhancement #3773: Optimize memory usage for directories with many files - Backing up a directory with hundreds of thousands or more files caused restic to require large - amounts of memory. We've now optimized the `backup` command such that it requires up to 30% less - memory. + Backing up a directory with hundreds of thousands or more files caused restic to + require large amounts of memory. We've now optimized the `backup` command such + that it requires up to 30% less memory. https://github.com/restic/restic/pull/3773 * Enhancement #3819: Validate include/exclude patterns before restoring Patterns provided to `restore` via `--exclude`, `--iexclude`, `--include` and - `--iinclude` weren't validated before running the restore. Invalid patterns would result in - error messages being printed repeatedly, and possibly unwanted files being restored. + `--iinclude` weren't validated before running the restore. Invalid patterns + would result in error messages being printed repeatedly, and possibly unwanted + files being restored. - Restic now validates all patterns before running the restore, and aborts with a fatal error if - an invalid pattern is detected. + Restic now validates all patterns before running the restore, and aborts with a + fatal error if an invalid pattern is detected. https://github.com/restic/restic/pull/3819 * Enhancement #3837: Improve SFTP repository initialization over slow links - The `init` command, when used on an SFTP backend, now sends multiple `mkdir` commands to the - backend concurrently. This reduces the waiting times when creating a repository over a very - slow connection. + The `init` command, when used on an SFTP backend, now sends multiple `mkdir` + commands to the backend concurrently. This reduces the waiting times when + creating a repository over a very slow connection. https://github.com/restic/restic/issues/3837 https://github.com/restic/restic/pull/3840 @@ -1644,9 +1789,9 @@ restic users. The changes are ordered by importance. * Bugfix #1106: Never lock repository for `list locks` - The `list locks` command previously locked to the repository by default. This had the problem - that it wouldn't work for an exclusively locked repository and that the command would also - display its own lock file which can be confusing. + The `list locks` command previously locked to the repository by default. This + had the problem that it wouldn't work for an exclusively locked repository and + that the command would also display its own lock file which can be confusing. Now, the `list locks` command never locks the repository. @@ -1655,22 +1800,24 @@ restic users. The changes are ordered by importance. * Bugfix #2345: Make cache crash-resistant and usable by multiple concurrent processes - The restic cache directory (`RESTIC_CACHE_DIR`) could end up in a broken state in the event of - restic (or the OS) crashing. This is now less likely to occur as files are downloaded to a - temporary location before being moved to their proper location. + The restic cache directory (`RESTIC_CACHE_DIR`) could end up in a broken state + in the event of restic (or the OS) crashing. This is now less likely to occur as + files are downloaded to a temporary location before being moved to their proper + location. - This also allows multiple concurrent restic processes to operate on a single repository - without conflicts. Previously, concurrent operations could cause segfaults because the - processes saw each other's partially downloaded files. + This also allows multiple concurrent restic processes to operate on a single + repository without conflicts. Previously, concurrent operations could cause + segfaults because the processes saw each other's partially downloaded files. https://github.com/restic/restic/issues/2345 https://github.com/restic/restic/pull/2838 * Bugfix #2452: Improve error handling of repository locking - Previously, when the lock refresh failed to delete the old lock file, it forgot about the newly - created one. Instead it continued trying to delete the old (usually no longer existing) lock - file and thus over time lots of lock files accumulated. This has now been fixed. + Previously, when the lock refresh failed to delete the old lock file, it forgot + about the newly created one. Instead it continued trying to delete the old + (usually no longer existing) lock file and thus over time lots of lock files + accumulated. This has now been fixed. https://github.com/restic/restic/issues/2452 https://github.com/restic/restic/issues/2473 @@ -1679,43 +1826,45 @@ restic users. The changes are ordered by importance. * Bugfix #2738: Don't print progress for `backup --json --quiet` - Unlike the text output, the `--json` output format still printed progress information even in - `--quiet` mode. This has now been fixed by always disabling the progress output in quiet mode. + Unlike the text output, the `--json` output format still printed progress + information even in `--quiet` mode. This has now been fixed by always disabling + the progress output in quiet mode. https://github.com/restic/restic/issues/2738 https://github.com/restic/restic/pull/3264 * Bugfix #3382: Make `check` command honor `RESTIC_CACHE_DIR` environment variable - Previously, the `check` command didn't honor the `RESTIC_CACHE_DIR` environment variable, - which caused problems in certain system/usage configurations. This has now been fixed. + Previously, the `check` command didn't honor the `RESTIC_CACHE_DIR` environment + variable, which caused problems in certain system/usage configurations. This has + now been fixed. https://github.com/restic/restic/issues/3382 https://github.com/restic/restic/pull/3474 * Bugfix #3488: `rebuild-index` failed if an index file was damaged - Previously, the `rebuild-index` command would fail with an error if an index file was damaged - or truncated. This has now been fixed. + Previously, the `rebuild-index` command would fail with an error if an index + file was damaged or truncated. This has now been fixed. - On older restic versions, a (slow) workaround is to use `rebuild-index --read-all-packs` or - to manually delete the damaged index. + On older restic versions, a (slow) workaround is to use `rebuild-index + --read-all-packs` or to manually delete the damaged index. https://github.com/restic/restic/pull/3488 * Bugfix #3518: Make `copy` command honor `--no-lock` for source repository - The `copy` command previously did not respect the `--no-lock` option for the source - repository, causing failures with read-only storage backends. This has now been fixed such - that the option is now respected. + The `copy` command previously did not respect the `--no-lock` option for the + source repository, causing failures with read-only storage backends. This has + now been fixed such that the option is now respected. https://github.com/restic/restic/issues/3518 https://github.com/restic/restic/pull/3589 * Bugfix #3556: Fix hang with Backblaze B2 on SSL certificate authority error - Previously, if a request failed with an SSL unknown certificate authority error, the B2 - backend retried indefinitely and restic would appear to hang. + Previously, if a request failed with an SSL unknown certificate authority error, + the B2 backend retried indefinitely and restic would appear to hang. This has now been fixed and restic instead fails with an error message. @@ -1725,95 +1874,103 @@ restic users. The changes are ordered by importance. * Bugfix #3591: Fix handling of `prune --max-repack-size=0` - Restic ignored the `--max-repack-size` option when passing a value of 0. This has now been - fixed. + Restic ignored the `--max-repack-size` option when passing a value of 0. This + has now been fixed. - As a workaround, `--max-repack-size=1` can be used with older versions of restic. + As a workaround, `--max-repack-size=1` can be used with older versions of + restic. https://github.com/restic/restic/pull/3591 * Bugfix #3601: Fix rclone backend prematurely exiting when receiving SIGINT on Windows - Previously, pressing Ctrl+C in a Windows console where restic was running with rclone as the - backend would cause rclone to exit prematurely due to getting a `SIGINT` signal at the same time - as restic. Restic would then wait for a long time for time with "unexpected EOF" and "rclone - stdio connection already closed" errors. + Previously, pressing Ctrl+C in a Windows console where restic was running with + rclone as the backend would cause rclone to exit prematurely due to getting a + `SIGINT` signal at the same time as restic. Restic would then wait for a long + time for time with "unexpected EOF" and "rclone stdio connection already closed" + errors. - This has now been fixed by restic starting the rclone process detached from the console restic - runs in (similar to starting processes in a new process group on Linux), which enables restic to - gracefully clean up rclone (which now never gets the `SIGINT`). + This has now been fixed by restic starting the rclone process detached from the + console restic runs in (similar to starting processes in a new process group on + Linux), which enables restic to gracefully clean up rclone (which now never gets + the `SIGINT`). https://github.com/restic/restic/issues/3601 https://github.com/restic/restic/pull/3602 * Bugfix #3619: Avoid choosing parent snapshots newer than time of new snapshot - The `backup` command, when a `--parent` was not provided, previously chose the most recent - matching snapshot as the parent snapshot. However, this didn't make sense when the user passed - `--time` to create a new snapshot older than the most recent snapshot. + The `backup` command, when a `--parent` was not provided, previously chose the + most recent matching snapshot as the parent snapshot. However, this didn't make + sense when the user passed `--time` to create a new snapshot older than the most + recent snapshot. - Instead, `backup` now chooses the most recent snapshot which is not newer than the - snapshot-being-created's timestamp, to avoid any time travel. + Instead, `backup` now chooses the most recent snapshot which is not newer than + the snapshot-being-created's timestamp, to avoid any time travel. https://github.com/restic/restic/pull/3619 * Bugfix #3667: The `mount` command now reports symlinks sizes - Symlinks used to have size zero in restic mountpoints, confusing some third-party tools. They - now have a size equal to the byte length of their target path, as required by POSIX. + Symlinks used to have size zero in restic mountpoints, confusing some + third-party tools. They now have a size equal to the byte length of their target + path, as required by POSIX. https://github.com/restic/restic/issues/3667 https://github.com/restic/restic/pull/3668 * Change #3519: Require Go 1.14 or newer - Restic now requires Go 1.14 to build. This allows it to use new standard library features - instead of an external dependency. + Restic now requires Go 1.14 to build. This allows it to use new standard library + features instead of an external dependency. https://github.com/restic/restic/issues/3519 * Change #3641: Ignore parent snapshot for `backup --stdin` - Restic uses a parent snapshot to speed up directory scanning when performing backups, but this - only wasted time and memory when the backup source is stdin (using the `--stdin` option of the - `backup` command), since no directory scanning is performed in this case. + Restic uses a parent snapshot to speed up directory scanning when performing + backups, but this only wasted time and memory when the backup source is stdin + (using the `--stdin` option of the `backup` command), since no directory + scanning is performed in this case. - Snapshots made with `backup --stdin` no longer have a parent snapshot, which allows restic to - skip some startup operations and saves a bit of resources. + Snapshots made with `backup --stdin` no longer have a parent snapshot, which + allows restic to skip some startup operations and saves a bit of resources. - The `--parent` option is still available for `backup --stdin`, but is now ignored. + The `--parent` option is still available for `backup --stdin`, but is now + ignored. https://github.com/restic/restic/issues/3641 https://github.com/restic/restic/pull/3645 * Enhancement #233: Support negative include/exclude patterns - If a pattern starts with an exclamation mark and it matches a file that was previously matched by - a regular pattern, the match is cancelled. Notably, this can be used with `--exclude-file` to - cancel the exclusion of some files. + If a pattern starts with an exclamation mark and it matches a file that was + previously matched by a regular pattern, the match is cancelled. Notably, this + can be used with `--exclude-file` to cancel the exclusion of some files. - It works similarly to `.gitignore`, with the same limitation; Once a directory is excluded, it - is not possible to include files inside the directory. + It works similarly to `.gitignore`, with the same limitation; Once a directory + is excluded, it is not possible to include files inside the directory. Example of use as an exclude pattern for the `backup` command: $HOME/**/* !$HOME/Documents !$HOME/code !$HOME/.emacs.d !$HOME/games # [...] - node_modules *~ *.o *.lo *.pyc # [...] $HOME/code/linux/* !$HOME/code/linux/.git # [...] + node_modules *~ *.o *.lo *.pyc # [...] $HOME/code/linux/* !$HOME/code/linux/.git + # [...] https://github.com/restic/restic/issues/233 https://github.com/restic/restic/pull/2311 * Enhancement #1542: Add `--dry-run`/`-n` option to `backup` command - Testing exclude filters and other configuration options was error prone as wrong filters - could cause files to be uploaded unintentionally. It was also not possible to estimate - beforehand how much data would be uploaded. + Testing exclude filters and other configuration options was error prone as wrong + filters could cause files to be uploaded unintentionally. It was also not + possible to estimate beforehand how much data would be uploaded. - The `backup` command now has a `--dry-run`/`-n` option, which performs all the normal steps of - a backup without actually writing anything to the repository. + The `backup` command now has a `--dry-run`/`-n` option, which performs all the + normal steps of a backup without actually writing anything to the repository. - Passing -vv will log information about files that would be added, allowing for verification of - source and exclusion options before running the real backup. + Passing -vv will log information about files that would be added, allowing for + verification of source and exclusion options before running the real backup. https://github.com/restic/restic/issues/1542 https://github.com/restic/restic/pull/2308 @@ -1822,14 +1979,14 @@ restic users. The changes are ordered by importance. * Enhancement #2202: Add upload checksum for Azure, GS, S3 and Swift backends - Previously only the B2 and partially the Swift backends verified the integrity of uploaded - (encrypted) files. The verification works by informing the backend about the expected hash of - the uploaded file. The backend then verifies the upload and thereby rules out any data - corruption during upload. + Previously only the B2 and partially the Swift backends verified the integrity + of uploaded (encrypted) files. The verification works by informing the backend + about the expected hash of the uploaded file. The backend then verifies the + upload and thereby rules out any data corruption during upload. - We have now added upload checksums for the Azure, GS, S3 and Swift backends, which besides - integrity checking for uploads also means that restic can now be used to store backups in S3 - buckets which have Object Lock enabled. + We have now added upload checksums for the Azure, GS, S3 and Swift backends, + which besides integrity checking for uploads also means that restic can now be + used to store backups in S3 buckets which have Object Lock enabled. https://github.com/restic/restic/issues/2202 https://github.com/restic/restic/issues/2700 @@ -1838,65 +1995,68 @@ restic users. The changes are ordered by importance. * Enhancement #2388: Add warning for S3 if partial credentials are provided - Previously restic did not notify about incomplete credentials when using the S3 backend, - instead just reporting access denied. + Previously restic did not notify about incomplete credentials when using the S3 + backend, instead just reporting access denied. - Restic now checks that both the AWS key ID and secret environment variables are set before - connecting to the remote server, and reports an error if not. + Restic now checks that both the AWS key ID and secret environment variables are + set before connecting to the remote server, and reports an error if not. https://github.com/restic/restic/issues/2388 https://github.com/restic/restic/pull/3532 * Enhancement #2508: Support JSON output and quiet mode for the `diff` command - The `diff` command now supports outputting machine-readable output in JSON format. To enable - this, pass the `--json` option to the command. To only print the summary and suppress detailed - output, pass the `--quiet` option. + The `diff` command now supports outputting machine-readable output in JSON + format. To enable this, pass the `--json` option to the command. To only print + the summary and suppress detailed output, pass the `--quiet` option. https://github.com/restic/restic/issues/2508 https://github.com/restic/restic/pull/3592 * Enhancement #2594: Speed up the `restore --verify` command - The `--verify` option lets the `restore` command verify the file content after it has restored - a snapshot. The performance of this operation has now been improved by up to a factor of two. + The `--verify` option lets the `restore` command verify the file content after + it has restored a snapshot. The performance of this operation has now been + improved by up to a factor of two. https://github.com/restic/restic/pull/2594 * Enhancement #2656: Add flag to disable TLS verification for self-signed certificates - There is now an `--insecure-tls` global option in restic, which disables TLS verification for - self-signed certificates in order to support some development workflows. + There is now an `--insecure-tls` global option in restic, which disables TLS + verification for self-signed certificates in order to support some development + workflows. https://github.com/restic/restic/issues/2656 https://github.com/restic/restic/pull/2657 * Enhancement #2816: The `backup` command no longer updates file access times on Linux - When reading files during backup, restic used to cause the operating system to update the - files' access times. Note that this did not apply to filesystems with disabled file access - times. + When reading files during backup, restic used to cause the operating system to + update the files' access times. Note that this did not apply to filesystems with + disabled file access times. - Restic now instructs the operating system not to update the file access time, if the user - running restic is the file owner or has root permissions. + Restic now instructs the operating system not to update the file access time, if + the user running restic is the file owner or has root permissions. https://github.com/restic/restic/pull/2816 * Enhancement #2880: Make `recover` collect only unreferenced trees - Previously, the `recover` command used to generate a snapshot containing *all* root trees, - even those which were already referenced by a snapshot. + Previously, the `recover` command used to generate a snapshot containing *all* + root trees, even those which were already referenced by a snapshot. - This has been improved such that it now only processes trees not already referenced by any - snapshot. + This has been improved such that it now only processes trees not already + referenced by any snapshot. https://github.com/restic/restic/pull/2880 * Enhancement #3003: Atomic uploads for the SFTP backend - The SFTP backend did not upload files atomically. An interrupted upload could leave an - incomplete file behind which could prevent restic from accessing the repository. This has now - been fixed and uploads in the SFTP backend are done atomically. + The SFTP backend did not upload files atomically. An interrupted upload could + leave an incomplete file behind which could prevent restic from accessing the + repository. This has now been fixed and uploads in the SFTP backend are done + atomically. https://github.com/restic/restic/issues/3003 https://github.com/restic/restic/pull/3524 @@ -1910,25 +2070,27 @@ restic users. The changes are ordered by importance. * Enhancement #3429: Verify that new or modified keys are stored correctly - When adding a new key or changing the password of a key, restic used to just create the new key (and - remove the old one, when changing the password). There was no verification that the new key was - stored correctly and works properly. As the repository cannot be decrypted without a valid key - file, this could in rare cases cause the repository to become inaccessible. + When adding a new key or changing the password of a key, restic used to just + create the new key (and remove the old one, when changing the password). There + was no verification that the new key was stored correctly and works properly. As + the repository cannot be decrypted without a valid key file, this could in rare + cases cause the repository to become inaccessible. - Restic now checks that new key files actually work before continuing. This can protect against - some (rare) cases of hardware or storage problems. + Restic now checks that new key files actually work before continuing. This can + protect against some (rare) cases of hardware or storage problems. https://github.com/restic/restic/pull/3429 * Enhancement #3436: Improve local backend's resilience to (system) crashes - Restic now ensures that files stored using the `local` backend are created atomically (that - is, files are either stored completely or not at all). This ensures that no incomplete files are - left behind even if restic is terminated while writing a file. + Restic now ensures that files stored using the `local` backend are created + atomically (that is, files are either stored completely or not at all). This + ensures that no incomplete files are left behind even if restic is terminated + while writing a file. - In addition, restic now tries to ensure that the directory in the repository which contains a - newly uploaded file is also written to disk. This can prevent missing files if the system - crashes or the disk is not properly unmounted. + In addition, restic now tries to ensure that the directory in the repository + which contains a newly uploaded file is also written to disk. This can prevent + missing files if the system crashes or the disk is not properly unmounted. https://github.com/restic/restic/pull/3436 @@ -1936,54 +2098,56 @@ restic users. The changes are ordered by importance. Restic used to silently ignore the `--no-lock` option of the `forget` command. - It now skips creation of lock file in case both `--dry-run` and `--no-lock` are specified. If - `--no-lock` option is specified without `--dry-run`, restic prints a warning message to - stderr. + It now skips creation of lock file in case both `--dry-run` and `--no-lock` are + specified. If `--no-lock` option is specified without `--dry-run`, restic prints + a warning message to stderr. https://github.com/restic/restic/issues/3464 https://github.com/restic/restic/pull/3623 * Enhancement #3490: Support random subset by size in `check --read-data-subset` - The `--read-data-subset` option of the `check` command now supports a third way of specifying - the subset to check, namely `nS` where `n` is a size in bytes with suffix `S` as k/K, m/M, g/G or - t/T. + The `--read-data-subset` option of the `check` command now supports a third way + of specifying the subset to check, namely `nS` where `n` is a size in bytes with + suffix `S` as k/K, m/M, g/G or t/T. https://github.com/restic/restic/issues/3490 https://github.com/restic/restic/pull/3548 * Enhancement #3508: Cache blobs read by the `dump` command - When dumping a file using the `dump` command, restic did not cache blobs in any way, so even - consecutive runs of the same blob were loaded from the repository again and again, slowing down - the dump. + When dumping a file using the `dump` command, restic did not cache blobs in any + way, so even consecutive runs of the same blob were loaded from the repository + again and again, slowing down the dump. - Now, the caching mechanism already used by the `fuse` command is also used by the `dump` - command. This makes dumping much faster, especially for sparse files. + Now, the caching mechanism already used by the `fuse` command is also used by + the `dump` command. This makes dumping much faster, especially for sparse files. https://github.com/restic/restic/pull/3508 * Enhancement #3511: Support configurable timeout for the rclone backend - A slow rclone backend could cause restic to time out while waiting for the repository to open. - Restic now offers an `-o rclone.timeout` option to make this timeout configurable. + A slow rclone backend could cause restic to time out while waiting for the + repository to open. Restic now offers an `-o rclone.timeout` option to make this + timeout configurable. https://github.com/restic/restic/issues/3511 https://github.com/restic/restic/pull/3514 * Enhancement #3541: Improve handling of temporary B2 delete errors - Deleting files on B2 could sometimes fail temporarily, which required restic to retry the - delete operation. In some cases the file was deleted nevertheless, causing the retries and - ultimately the restic command to fail. This has now been fixed. + Deleting files on B2 could sometimes fail temporarily, which required restic to + retry the delete operation. In some cases the file was deleted nevertheless, + causing the retries and ultimately the restic command to fail. This has now been + fixed. https://github.com/restic/restic/issues/3541 https://github.com/restic/restic/pull/3544 * Enhancement #3542: Add file mode in symbolic notation to `ls --json` - The `ls --json` command now provides the file mode in symbolic notation (using the - `permissions` key), aligned with `find --json`. + The `ls --json` command now provides the file mode in symbolic notation (using + the `permissions` key), aligned with `find --json`. https://github.com/restic/restic/issues/3542 https://github.com/restic/restic/pull/3573 @@ -1991,11 +2155,12 @@ restic users. The changes are ordered by importance. * Enhancement #3593: Improve `copy` performance by parallelizing IO - Restic copy previously only used a single thread for copying blobs between repositories, - which resulted in limited performance when copying small blobs to/from a high latency backend - (i.e. any remote backend, especially b2). + Restic copy previously only used a single thread for copying blobs between + repositories, which resulted in limited performance when copying small blobs + to/from a high latency backend (i.e. any remote backend, especially b2). - Copying will now use 8 parallel threads to increase the throughput of the copy operation. + Copying will now use 8 parallel threads to increase the throughput of the copy + operation. https://github.com/restic/restic/pull/3593 @@ -2033,9 +2198,9 @@ restic users. The changes are ordered by importance. * Bugfix #2742: Improve error handling for rclone and REST backend over HTTP2 - When retrieving data from the rclone / REST backend while also using HTTP2 restic did not detect - when no data was returned at all. This could cause for example the `check` command to report the - following error: + When retrieving data from the rclone / REST backend while also using HTTP2 + restic did not detect when no data was returned at all. This could cause for + example the `check` command to report the following error: Pack ID does not match, want [...], got e3b0c442 @@ -2047,98 +2212,105 @@ restic users. The changes are ordered by importance. * Bugfix #3111: Fix terminal output redirection for PowerShell - When redirecting the output of restic using PowerShell on Windows, the output contained - terminal escape characters. This has been fixed by properly detecting the terminal type. + When redirecting the output of restic using PowerShell on Windows, the output + contained terminal escape characters. This has been fixed by properly detecting + the terminal type. - In addition, the mintty terminal now shows progress output for the backup command. + In addition, the mintty terminal now shows progress output for the backup + command. https://github.com/restic/restic/issues/3111 https://github.com/restic/restic/pull/3325 * Bugfix #3184: `backup --quiet` no longer prints status information - A regression in the latest restic version caused the output of `backup --quiet` to contain - large amounts of backup progress information when run using an interactive terminal. This is - fixed now. + A regression in the latest restic version caused the output of `backup --quiet` + to contain large amounts of backup progress information when run using an + interactive terminal. This is fixed now. - A workaround for this bug is to run restic as follows: `restic backup --quiet [..] | cat -`. + A workaround for this bug is to run restic as follows: `restic backup --quiet + [..] | cat -`. https://github.com/restic/restic/issues/3184 https://github.com/restic/restic/pull/3186 * Bugfix #3214: Treat an empty password as a fatal error for repository init - When attempting to initialize a new repository, if an empty password was supplied, the - repository would be created but the init command would return an error with a stack trace. Now, - if an empty password is provided, it is treated as a fatal error, and no repository is created. + When attempting to initialize a new repository, if an empty password was + supplied, the repository would be created but the init command would return an + error with a stack trace. Now, if an empty password is provided, it is treated + as a fatal error, and no repository is created. https://github.com/restic/restic/issues/3214 https://github.com/restic/restic/pull/3283 * Bugfix #3267: `copy` failed to copy snapshots in rare cases - The `copy` command could in rare cases fail with the error message `SaveTree(...) returned - unexpected id ...`. This has been fixed. + The `copy` command could in rare cases fail with the error message + `SaveTree(...) returned unexpected id ...`. This has been fixed. - On Linux/BSDs, the error could be caused by backing up symlinks with non-UTF-8 target paths. - Note that, due to limitations in the repository format, these are not stored properly and - should be avoided if possible. + On Linux/BSDs, the error could be caused by backing up symlinks with non-UTF-8 + target paths. Note that, due to limitations in the repository format, these are + not stored properly and should be avoided if possible. https://github.com/restic/restic/issues/3267 https://github.com/restic/restic/pull/3310 * Bugfix #3296: Fix crash of `check --read-data-subset=x%` run for an empty repository - The command `restic check --read-data-subset=x%` crashed when run for an empty repository. - This has been fixed. + The command `restic check --read-data-subset=x%` crashed when run for an empty + repository. This has been fixed. https://github.com/restic/restic/issues/3296 https://github.com/restic/restic/pull/3309 * Bugfix #3302: Fix `fdopendir: not a directory` error for local backend - The `check`, `list packs`, `prune` and `rebuild-index` commands failed for the local backend - when the `data` folder in the repository contained files. This has been fixed. + The `check`, `list packs`, `prune` and `rebuild-index` commands failed for the + local backend when the `data` folder in the repository contained files. This has + been fixed. https://github.com/restic/restic/issues/3302 https://github.com/restic/restic/pull/3308 * Bugfix #3305: Fix possibly missing backup summary of JSON output in case of error - When using `--json` output it happened from time to time that the summary output was missing in - case an error occurred. This has been fixed. + When using `--json` output it happened from time to time that the summary output + was missing in case an error occurred. This has been fixed. https://github.com/restic/restic/pull/3305 * Bugfix #3334: Print `created new cache` message only on a terminal - The message `created new cache` was printed even when the output wasn't a terminal. That broke - piping `restic dump` output to tar or zip if cache directory didn't exist. The message is now - only printed on a terminal. + The message `created new cache` was printed even when the output wasn't a + terminal. That broke piping `restic dump` output to tar or zip if cache + directory didn't exist. The message is now only printed on a terminal. https://github.com/restic/restic/issues/3334 https://github.com/restic/restic/pull/3343 * Bugfix #3380: Fix crash of `backup --exclude='**'` - The exclude filter `**`, which excludes all files, caused restic to crash. This has been - corrected. + The exclude filter `**`, which excludes all files, caused restic to crash. This + has been corrected. https://github.com/restic/restic/issues/3380 https://github.com/restic/restic/pull/3393 * Bugfix #3439: Correctly handle download errors during `restore` - Due to a regression in restic 0.12.0, the `restore` command in some cases did not retry download - errors and only printed a warning. This has been fixed by retrying incomplete data downloads. + Due to a regression in restic 0.12.0, the `restore` command in some cases did + not retry download errors and only printed a warning. This has been fixed by + retrying incomplete data downloads. https://github.com/restic/restic/issues/3439 https://github.com/restic/restic/pull/3449 * Change #3247: Empty files now have size of 0 in `ls --json` output - The `ls --json` command used to omit the sizes of empty files in its output. It now reports a size - of zero explicitly for regular files, while omitting the size field for all other types. + The `ls --json` command used to omit the sizes of empty files in its output. It + now reports a size of zero explicitly for regular files, while omitting the size + field for all other types. https://github.com/restic/restic/issues/3247 https://github.com/restic/restic/pull/3257 @@ -2152,9 +2324,9 @@ restic users. The changes are ordered by importance. * Enhancement #3167: Allow specifying limit of `snapshots` list - The `--last` option allowed limiting the output of the `snapshots` command to the latest - snapshot for each host. The new `--latest n` option allows limiting the output to the latest `n` - snapshots. + The `--last` option allowed limiting the output of the `snapshots` command to + the latest snapshot for each host. The new `--latest n` option allows limiting + the output to the latest `n` snapshots. This change deprecates the option `--last` in favour of `--latest 1`. @@ -2162,13 +2334,15 @@ restic users. The changes are ordered by importance. * Enhancement #3293: Add `--repository-file2` option to `init` and `copy` command - The `init` and `copy` command can now be used with the `--repository-file2` option or the - `$RESTIC_REPOSITORY_FILE2` environment variable. These to options are in addition to the - `--repo2` flag and allow you to read the destination repository from a file. + The `init` and `copy` command can now be used with the `--repository-file2` + option or the `$RESTIC_REPOSITORY_FILE2` environment variable. These to options + are in addition to the `--repo2` flag and allow you to read the destination + repository from a file. - Using both `--repository-file` and `--repo2` options resulted in an error for the `copy` or - `init` command. The handling of this combination of options has been fixed. A workaround for - this issue is to only use `--repo` or `-r` and `--repo2` for `init` or `copy`. + Using both `--repository-file` and `--repo2` options resulted in an error for + the `copy` or `init` command. The handling of this combination of options has + been fixed. A workaround for this issue is to only use `--repo` or `-r` and + `--repo2` for `init` or `copy`. https://github.com/restic/restic/issues/3293 https://github.com/restic/restic/pull/3294 @@ -2181,9 +2355,9 @@ restic users. The changes are ordered by importance. * Enhancement #3336: SFTP backend now checks for disk space - Backing up over SFTP previously spewed multiple generic "failure" messages when the remote - disk was full. It now checks for disk space before writing a file and fails immediately with a "no - space left on device" message. + Backing up over SFTP previously spewed multiple generic "failure" messages when + the remote disk was full. It now checks for disk space before writing a file and + fails immediately with a "no space left on device" message. https://github.com/restic/restic/issues/3336 https://github.com/restic/restic/pull/3345 @@ -2197,15 +2371,17 @@ restic users. The changes are ordered by importance. * Enhancement #3414: Add `--keep-within-hourly` option to restic forget - The `forget` command allowed keeping a given number of hourly backups or to keep all backups - within a given interval, but it was not possible to specify keeping hourly backups within a - given interval. + The `forget` command allowed keeping a given number of hourly backups or to keep + all backups within a given interval, but it was not possible to specify keeping + hourly backups within a given interval. - The new `--keep-within-hourly` option now offers this functionality. Similar options for - daily/weekly/monthly/yearly are also implemented, the new options are: + The new `--keep-within-hourly` option now offers this functionality. Similar + options for daily/weekly/monthly/yearly are also implemented, the new options + are: - --keep-within-hourly <1y2m3d4h> --keep-within-daily <1y2m3d4h> --keep-within-weekly - <1y2m3d4h> --keep-within-monthly <1y2m3d4h> --keep-within-yearly <1y2m3d4h> + --keep-within-hourly <1y2m3d4h> --keep-within-daily <1y2m3d4h> + --keep-within-weekly <1y2m3d4h> --keep-within-monthly <1y2m3d4h> + --keep-within-yearly <1y2m3d4h> https://github.com/restic/restic/issues/3414 https://github.com/restic/restic/pull/3416 @@ -2213,30 +2389,32 @@ restic users. The changes are ordered by importance. * Enhancement #3426: Optimize read performance of mount command - Reading large files in a mounted repository may be up to five times faster. This improvement - primarily applies to repositories stored at a backend that can be accessed with low latency, - like e.g. the local backend. + Reading large files in a mounted repository may be up to five times faster. This + improvement primarily applies to repositories stored at a backend that can be + accessed with low latency, like e.g. the local backend. https://github.com/restic/restic/pull/3426 * Enhancement #3427: `find --pack` fallback to index if data file is missing - When investigating a repository with missing data files, it might be useful to determine - affected snapshots before running `rebuild-index`. Previously, `find --pack pack-id` - returned no data as it required accessing the data file. Now, if the necessary data is still - available in the repository index, it gets retrieved from there. + When investigating a repository with missing data files, it might be useful to + determine affected snapshots before running `rebuild-index`. Previously, `find + --pack pack-id` returned no data as it required accessing the data file. Now, if + the necessary data is still available in the repository index, it gets retrieved + from there. - The command now also supports looking up multiple pack files in a single `find` run. + The command now also supports looking up multiple pack files in a single `find` + run. https://github.com/restic/restic/pull/3427 https://forum.restic.net/t/missing-packs-not-found/2600 * Enhancement #3456: Support filtering and specifying untagged snapshots - It was previously not possible to specify an empty tag with the `--tag` and `--keep-tag` - options. This has now been fixed, such that `--tag ''` and `--keep-tag ''` now matches - snapshots without tags. This allows e.g. the `snapshots` and `forget` commands to only - operate on untagged snapshots. + It was previously not possible to specify an empty tag with the `--tag` and + `--keep-tag` options. This has now been fixed, such that `--tag ''` and + `--keep-tag ''` now matches snapshots without tags. This allows e.g. the + `snapshots` and `forget` commands to only operate on untagged snapshots. https://github.com/restic/restic/issues/3456 https://github.com/restic/restic/pull/3457 @@ -2288,28 +2466,28 @@ restic users. The changes are ordered by importance. * Bugfix #1681: Make `mount` not create missing mount point directory - When specifying a non-existent directory as mount point for the `mount` command, restic used - to create the specified directory automatically. + When specifying a non-existent directory as mount point for the `mount` command, + restic used to create the specified directory automatically. - This has now changed such that restic instead gives an error when the specified directory for - the mount point does not exist. + This has now changed such that restic instead gives an error when the specified + directory for the mount point does not exist. https://github.com/restic/restic/issues/1681 https://github.com/restic/restic/pull/3008 * Bugfix #1800: Ignore `no data available` filesystem error during backup - Restic was unable to backup files on some filesystems, for example certain configurations of - CIFS on Linux which return a `no data available` error when reading extended attributes. These - errors are now ignored. + Restic was unable to backup files on some filesystems, for example certain + configurations of CIFS on Linux which return a `no data available` error when + reading extended attributes. These errors are now ignored. https://github.com/restic/restic/issues/1800 https://github.com/restic/restic/pull/3034 * Bugfix #2563: Report the correct owner of directories in FUSE mounts - Restic 0.10.0 changed the FUSE mount to always report the current user as the owner of - directories within the FUSE mount, which is incorrect. + Restic 0.10.0 changed the FUSE mount to always report the current user as the + owner of directories within the FUSE mount, which is incorrect. This is now changed back to reporting the correct owner of a directory. @@ -2318,30 +2496,31 @@ restic users. The changes are ordered by importance. * Bugfix #2688: Make `backup` and `tag` commands separate tags by comma - Running `restic backup --tag foo,bar` previously created snapshots with one single tag - containing a comma (`foo,bar`) instead of two tags (`foo`, `bar`). + Running `restic backup --tag foo,bar` previously created snapshots with one + single tag containing a comma (`foo,bar`) instead of two tags (`foo`, `bar`). - Similarly, the `tag` command's `--set`, `--add` and `--remove` options would treat - `foo,bar` as one tag instead of two tags. This was inconsistent with other commands and often - unexpected when one intended `foo,bar` to mean two tags. + Similarly, the `tag` command's `--set`, `--add` and `--remove` options would + treat `foo,bar` as one tag instead of two tags. This was inconsistent with other + commands and often unexpected when one intended `foo,bar` to mean two tags. - To be consistent in all commands, restic now interprets `foo,bar` to mean two separate tags - (`foo` and `bar`) instead of one tag (`foo,bar`) everywhere, including in the `backup` and - `tag` commands. + To be consistent in all commands, restic now interprets `foo,bar` to mean two + separate tags (`foo` and `bar`) instead of one tag (`foo,bar`) everywhere, + including in the `backup` and `tag` commands. - NOTE: This change might result in unexpected behavior in cases where you use the `forget` - command and filter on tags like `foo,bar`. Snapshots previously backed up with `--tag - foo,bar` will still not match that filter, but snapshots saved from now on will match that - filter. + NOTE: This change might result in unexpected behavior in cases where you use the + `forget` command and filter on tags like `foo,bar`. Snapshots previously backed + up with `--tag foo,bar` will still not match that filter, but snapshots saved + from now on will match that filter. - To replace `foo,bar` tags with `foo` and `bar` tags in old snapshots, you can first generate a - list of the relevant snapshots using a command like: + To replace `foo,bar` tags with `foo` and `bar` tags in old snapshots, you can + first generate a list of the relevant snapshots using a command like: - Restic snapshots --json --quiet | jq '.[] | select(contains({tags: ["foo,bar"]})) | .id' + Restic snapshots --json --quiet | jq '.[] | select(contains({tags: + ["foo,bar"]})) | .id' - And then use `restic tag --set foo --set bar snapshotID [...]` to set the new tags. Please adjust - the commands to include real tag names and any additional tags, as well as the list of snapshots - to process. + And then use `restic tag --set foo --set bar snapshotID [...]` to set the new + tags. Please adjust the commands to include real tag names and any additional + tags, as well as the list of snapshots to process. https://github.com/restic/restic/issues/2688 https://github.com/restic/restic/pull/2690 @@ -2355,14 +2534,14 @@ restic users. The changes are ordered by importance. * Bugfix #3014: Fix sporadic stream reset between rclone and restic - Sometimes when using restic with the `rclone` backend, an error message similar to the - following would be printed: + Sometimes when using restic with the `rclone` backend, an error message similar + to the following would be printed: Didn't finish writing GET request (wrote 0/xxx): http2: stream closed - It was found that this was caused by restic closing the connection to rclone to soon when - downloading data. A workaround has been added which waits for the end of the download before - closing the connection. + It was found that this was caused by restic closing the connection to rclone to + soon when downloading data. A workaround has been added which waits for the end + of the download before closing the connection. https://github.com/rclone/rclone/issues/2598 https://github.com/restic/restic/pull/3014 @@ -2380,125 +2559,130 @@ restic users. The changes are ordered by importance. * Bugfix #3100: Do not require gs bucket permissions when running `init` - Restic used to require bucket level permissions for the `gs` backend in order to initialize a - restic repository. + Restic used to require bucket level permissions for the `gs` backend in order to + initialize a restic repository. - It now allows a `gs` service account to initialize a repository if the bucket does exist and the - service account has permissions to write/read to that bucket. + It now allows a `gs` service account to initialize a repository if the bucket + does exist and the service account has permissions to write/read to that bucket. https://github.com/restic/restic/issues/3100 * Bugfix #3111: Correctly detect output redirection for `backup` command on Windows - On Windows, since restic 0.10.0 the `backup` command did not properly detect when the output - was redirected to a file. This caused restic to output terminal control characters. This has - been fixed by correcting the terminal detection. + On Windows, since restic 0.10.0 the `backup` command did not properly detect + when the output was redirected to a file. This caused restic to output terminal + control characters. This has been fixed by correcting the terminal detection. https://github.com/restic/restic/issues/3111 https://github.com/restic/restic/pull/3150 * Bugfix #3151: Don't create invalid snapshots when `backup` is interrupted - When canceling a backup run at a certain moment it was possible that restic created a snapshot - with an invalid "null" tree. This caused `check` and other operations to fail. The `backup` - command now properly handles interruptions and never saves a snapshot when interrupted. + When canceling a backup run at a certain moment it was possible that restic + created a snapshot with an invalid "null" tree. This caused `check` and other + operations to fail. The `backup` command now properly handles interruptions and + never saves a snapshot when interrupted. https://github.com/restic/restic/issues/3151 https://github.com/restic/restic/pull/3164 * Bugfix #3152: Do not hang until foregrounded when completed in background - On Linux, when running in the background restic failed to stop the terminal output of the - `backup` command after it had completed. This caused restic to hang until moved to the - foreground. This has now been fixed. + On Linux, when running in the background restic failed to stop the terminal + output of the `backup` command after it had completed. This caused restic to + hang until moved to the foreground. This has now been fixed. https://github.com/restic/restic/pull/3152 https://forum.restic.net/t/restic-alpine-container-cron-hangs-epoll-pwait/3334 * Bugfix #3166: Improve error handling in the `restore` command - The `restore` command used to not print errors while downloading file contents from the - repository. It also incorrectly exited with a zero error code even when there were errors - during the restore process. This has all been fixed and `restore` now returns with a non-zero - exit code when there's an error. + The `restore` command used to not print errors while downloading file contents + from the repository. It also incorrectly exited with a zero error code even when + there were errors during the restore process. This has all been fixed and + `restore` now returns with a non-zero exit code when there's an error. https://github.com/restic/restic/issues/3166 https://github.com/restic/restic/pull/3207 * Bugfix #3232: Correct statistics for overlapping targets - A user reported that restic's statistics and progress information during backup was not - correctly calculated when the backup targets (files/dirs to save) overlap. For example, - consider a directory `foo` which contains (among others) a file `foo/bar`. When `restic - backup foo foo/bar` was run, restic counted the size of the file `foo/bar` twice, so the - completeness percentage as well as the number of files was wrong. This is now corrected. + A user reported that restic's statistics and progress information during backup + was not correctly calculated when the backup targets (files/dirs to save) + overlap. For example, consider a directory `foo` which contains (among others) a + file `foo/bar`. When `restic backup foo foo/bar` was run, restic counted the + size of the file `foo/bar` twice, so the completeness percentage as well as the + number of files was wrong. This is now corrected. https://github.com/restic/restic/issues/3232 https://github.com/restic/restic/pull/3243 * Bugfix #3249: Improve error handling in `gs` backend - The `gs` backend did not notice when the last step of completing a file upload failed. Under rare - circumstances, this could cause missing files in the backup repository. This has now been - fixed. + The `gs` backend did not notice when the last step of completing a file upload + failed. Under rare circumstances, this could cause missing files in the backup + repository. This has now been fixed. https://github.com/restic/restic/pull/3249 * Change #3095: Deleting files on Google Drive now moves them to the trash - When deleting files on Google Drive via the `rclone` backend, restic used to bypass the trash - folder required that one used the `-o rclone.args` option to enable usage of the trash folder. - This ensured that deleted files in Google Drive were not kept indefinitely in the trash folder. - However, since Google Drive's trash retention policy changed to deleting trashed files after - 30 days, this is no longer needed. + When deleting files on Google Drive via the `rclone` backend, restic used to + bypass the trash folder required that one used the `-o rclone.args` option to + enable usage of the trash folder. This ensured that deleted files in Google + Drive were not kept indefinitely in the trash folder. However, since Google + Drive's trash retention policy changed to deleting trashed files after 30 days, + this is no longer needed. - Restic now leaves it up to rclone and its configuration to use or not use the trash folder when - deleting files. The default is to use the trash folder, as of rclone 1.53.2. To re-enable the - restic 0.11 behavior, set the `RCLONE_DRIVE_USE_TRASH` environment variable or change the - rclone configuration. See the rclone documentation for more details. + Restic now leaves it up to rclone and its configuration to use or not use the + trash folder when deleting files. The default is to use the trash folder, as of + rclone 1.53.2. To re-enable the restic 0.11 behavior, set the + `RCLONE_DRIVE_USE_TRASH` environment variable or change the rclone + configuration. See the rclone documentation for more details. https://github.com/restic/restic/issues/3095 https://github.com/restic/restic/pull/3102 * Enhancement #909: Back up mountpoints as empty directories - When the `--one-file-system` option is specified to `restic backup`, it ignores all file - systems mounted below one of the target directories. This means that when a snapshot is - restored, users needed to manually recreate the mountpoint directories. + When the `--one-file-system` option is specified to `restic backup`, it ignores + all file systems mounted below one of the target directories. This means that + when a snapshot is restored, users needed to manually recreate the mountpoint + directories. - Restic now backs up mountpoints as empty directories and therefore implements the same - approach as `tar`. + Restic now backs up mountpoints as empty directories and therefore implements + the same approach as `tar`. https://github.com/restic/restic/issues/909 https://github.com/restic/restic/pull/3119 * Enhancement #2186: Allow specifying percentage in `check --read-data-subset` - We've enhanced the `check` command's `--read-data-subset` option to also accept a - percentage (e.g. `2.5%` or `10%`). This will check the given percentage of pack files (which - are randomly selected on each run). + We've enhanced the `check` command's `--read-data-subset` option to also accept + a percentage (e.g. `2.5%` or `10%`). This will check the given percentage of + pack files (which are randomly selected on each run). https://github.com/restic/restic/issues/2186 https://github.com/restic/restic/pull/3038 * Enhancement #2433: Make the `dump` command support `zip` format - Previously, restic could dump the contents of a whole folder structure only in the `tar` - format. The `dump` command now has a new flag to change output format to `zip`. Just pass - `--archive zip` as an option to `restic dump`. + Previously, restic could dump the contents of a whole folder structure only in + the `tar` format. The `dump` command now has a new flag to change output format + to `zip`. Just pass `--archive zip` as an option to `restic dump`. https://github.com/restic/restic/pull/2433 https://github.com/restic/restic/pull/3081 * Enhancement #2453: Report permanent/fatal backend errors earlier - When encountering errors in reading from or writing to storage backends, restic retries the - failing operation up to nine times (for a total of ten attempts). It used to retry all backend - operations, but now detects some permanent error conditions so that it can report fatal errors - earlier. + When encountering errors in reading from or writing to storage backends, restic + retries the failing operation up to nine times (for a total of ten attempts). It + used to retry all backend operations, but now detects some permanent error + conditions so that it can report fatal errors earlier. - Permanent failures include local disks being full, SSH connections dropping and permission - errors. + Permanent failures include local disks being full, SSH connections dropping and + permission errors. https://github.com/restic/restic/issues/2453 https://github.com/restic/restic/issues/3180 @@ -2507,23 +2691,26 @@ restic users. The changes are ordered by importance. * Enhancement #2495: Add option to let `backup` trust mtime without checking ctime - The `backup` command used to require that both `ctime` and `mtime` of a file matched with a - previously backed up version to determine that the file was unchanged. In other words, if - either `ctime` or `mtime` of the file had changed, it would be considered changed and restic - would read the file's content again to back up the relevant (changed) parts of it. + The `backup` command used to require that both `ctime` and `mtime` of a file + matched with a previously backed up version to determine that the file was + unchanged. In other words, if either `ctime` or `mtime` of the file had changed, + it would be considered changed and restic would read the file's content again to + back up the relevant (changed) parts of it. - The new option `--ignore-ctime` makes restic look at `mtime` only, such that `ctime` changes - for a file does not cause restic to read the file's contents again. + The new option `--ignore-ctime` makes restic look at `mtime` only, such that + `ctime` changes for a file does not cause restic to read the file's contents + again. - The check for both `ctime` and `mtime` was introduced in restic 0.9.6 to make backups more - reliable in the face of programs that reset `mtime` (some Unix archivers do that), but it turned - out to often be expensive because it made restic read file contents even if only the metadata - (owner, permissions) of a file had changed. The new `--ignore-ctime` option lets the user - restore the 0.9.5 behavior when needed. The existing `--ignore-inode` option already turned + The check for both `ctime` and `mtime` was introduced in restic 0.9.6 to make + backups more reliable in the face of programs that reset `mtime` (some Unix + archivers do that), but it turned out to often be expensive because it made + restic read file contents even if only the metadata (owner, permissions) of a + file had changed. The new `--ignore-ctime` option lets the user restore the + 0.9.5 behavior when needed. The existing `--ignore-inode` option already turned off this behavior, but also removed a different check. - Please note that changes in files' metadata are still recorded, regardless of the command line - options provided to the backup command. + Please note that changes in files' metadata are still recorded, regardless of + the command line options provided to the backup command. https://github.com/restic/restic/issues/2495 https://github.com/restic/restic/issues/2558 @@ -2532,20 +2719,21 @@ restic users. The changes are ordered by importance. * Enhancement #2528: Add Alibaba/Aliyun OSS support in the `s3` backend - A new extended option `s3.bucket-lookup` has been added to support Alibaba/Aliyun OSS in the - `s3` backend. The option can be set to one of the following values: + A new extended option `s3.bucket-lookup` has been added to support + Alibaba/Aliyun OSS in the `s3` backend. The option can be set to one of the + following values: - - `auto` - Existing behaviour - `dns` - Use DNS style bucket access - `path` - Use path style - bucket access + - `auto` - Existing behaviour - `dns` - Use DNS style bucket access - `path` - + Use path style bucket access - To make the `s3` backend work with Alibaba/Aliyun OSS you must set `s3.bucket-lookup` to `dns` - and set the `s3.region` parameter. For example: + To make the `s3` backend work with Alibaba/Aliyun OSS you must set + `s3.bucket-lookup` to `dns` and set the `s3.region` parameter. For example: Restic -o s3.bucket-lookup=dns -o s3.region=oss-eu-west-1 -r s3:https://oss-eu-west-1.aliyuncs.com/bucketname init - Note that `s3.region` must be set, otherwise the MinIO SDK tries to look it up and it seems that - Alibaba doesn't support that properly. + Note that `s3.region` must be set, otherwise the MinIO SDK tries to look it up + and it seems that Alibaba doesn't support that properly. https://github.com/restic/restic/issues/2528 https://github.com/restic/restic/pull/2535 @@ -2554,14 +2742,14 @@ restic users. The changes are ordered by importance. The `backup`, `check` and `prune` commands never printed any progress reports on non-interactive terminals. This behavior is now configurable using the - `RESTIC_PROGRESS_FPS` environment variable. Use for example a value of `1` for an update - every second, or `0.01666` for an update every minute. + `RESTIC_PROGRESS_FPS` environment variable. Use for example a value of `1` for + an update every second, or `0.01666` for an update every minute. - The `backup` command now also prints the current progress when restic receives a `SIGUSR1` - signal. + The `backup` command now also prints the current progress when restic receives a + `SIGUSR1` signal. - Setting the `RESTIC_PROGRESS_FPS` environment variable or sending a `SIGUSR1` signal - prints a status report even when `--quiet` was specified. + Setting the `RESTIC_PROGRESS_FPS` environment variable or sending a `SIGUSR1` + signal prints a status report even when `--quiet` was specified. https://github.com/restic/restic/issues/2706 https://github.com/restic/restic/issues/3194 @@ -2569,21 +2757,22 @@ restic users. The changes are ordered by importance. * Enhancement #2718: Improve `prune` performance and make it more customizable - The `prune` command is now much faster. This is especially the case for remote repositories or - repositories with not much data to remove. Also the memory usage of the `prune` command is now - reduced. + The `prune` command is now much faster. This is especially the case for remote + repositories or repositories with not much data to remove. Also the memory usage + of the `prune` command is now reduced. - Restic used to rebuild the index from scratch after pruning. This could lead to missing packs in - the index in some cases for eventually consistent backends such as e.g. AWS S3. This behavior is - now changed and the index rebuilding uses the information already known by `prune`. + Restic used to rebuild the index from scratch after pruning. This could lead to + missing packs in the index in some cases for eventually consistent backends such + as e.g. AWS S3. This behavior is now changed and the index rebuilding uses the + information already known by `prune`. - By default, the `prune` command no longer removes all unused data. This behavior can be - fine-tuned by new options, like the acceptable amount of unused space or the maximum size of - data to reorganize. For more details, please see + By default, the `prune` command no longer removes all unused data. This behavior + can be fine-tuned by new options, like the acceptable amount of unused space or + the maximum size of data to reorganize. For more details, please see https://restic.readthedocs.io/en/stable/060_forget.html . - Moreover, `prune` now accepts the `--dry-run` option and also running `forget --dry-run - --prune` will show what `prune` would do. + Moreover, `prune` now accepts the `--dry-run` option and also running `forget + --dry-run --prune` will show what `prune` would do. This enhancement also fixes several open issues, e.g.: - https://github.com/restic/restic/issues/1140 - @@ -2598,68 +2787,74 @@ restic users. The changes are ordered by importance. * Enhancement #2941: Speed up the repacking step of the `prune` command - The repack step of the `prune` command, which moves still used file parts into new pack files - such that the old ones can be garbage collected later on, now processes multiple pack files in - parallel. This is especially beneficial for high latency backends or when using a fast network - connection. + The repack step of the `prune` command, which moves still used file parts into + new pack files such that the old ones can be garbage collected later on, now + processes multiple pack files in parallel. This is especially beneficial for + high latency backends or when using a fast network connection. https://github.com/restic/restic/pull/2941 * Enhancement #2944: Add `backup` options `--files-from-{verbatim,raw}` - The new `backup` options `--files-from-verbatim` and `--files-from-raw` read a list of - files to back up from a file. Unlike the existing `--files-from` option, these options do not - interpret the listed filenames as glob patterns; instead, whitespace in filenames is - preserved as-is and no pattern expansion is done. Please see the documentation for specifics. + The new `backup` options `--files-from-verbatim` and `--files-from-raw` read a + list of files to back up from a file. Unlike the existing `--files-from` option, + these options do not interpret the listed filenames as glob patterns; instead, + whitespace in filenames is preserved as-is and no pattern expansion is done. + Please see the documentation for specifics. - These new options are highly recommended over `--files-from`, when using a script to generate - the list of files to back up. + These new options are highly recommended over `--files-from`, when using a + script to generate the list of files to back up. https://github.com/restic/restic/issues/2944 https://github.com/restic/restic/issues/3013 * Enhancement #3006: Speed up the `rebuild-index` command - We've optimized the `rebuild-index` command. Now, existing index entries are used to - minimize the number of pack files that must be read. This speeds up the index rebuild a lot. + We've optimized the `rebuild-index` command. Now, existing index entries are + used to minimize the number of pack files that must be read. This speeds up the + index rebuild a lot. - Additionally, the option `--read-all-packs` has been added, implementing the previous - behavior. + Additionally, the option `--read-all-packs` has been added, implementing the + previous behavior. https://github.com/restic/restic/pull/3006 https://github.com/restic/restic/issue/2547 * Enhancement #3048: Add more checks for index and pack files in the `check` command - The `check` command run with the `--read-data` or `--read-data-subset` options used to only - verify only the pack file content - it did not check if the blobs within the pack are correctly - contained in the index. + The `check` command run with the `--read-data` or `--read-data-subset` options + used to only verify only the pack file content - it did not check if the blobs + within the pack are correctly contained in the index. A check for the latter is now in place, which can print the following error: Blob ID is not contained in index or position is incorrect - Another test is also added, which compares pack file sizes computed from the index and the pack - header with the actual file size. This test is able to detect truncated pack files. + Another test is also added, which compares pack file sizes computed from the + index and the pack header with the actual file size. This test is able to detect + truncated pack files. - If the index is not correct, it can be rebuilt by using the `rebuild-index` command. + If the index is not correct, it can be rebuilt by using the `rebuild-index` + command. - Having added these tests, `restic check` is now able to detect non-existing blobs which are - wrongly referenced in the index. This situation could have lead to missing data. + Having added these tests, `restic check` is now able to detect non-existing + blobs which are wrongly referenced in the index. This situation could have lead + to missing data. https://github.com/restic/restic/pull/3048 https://github.com/restic/restic/pull/3082 * Enhancement #3083: Allow usage of deprecated S3 `ListObjects` API - Some S3 API implementations, e.g. Ceph before version 14.2.5, have a broken `ListObjectsV2` - implementation which causes problems for restic when using their API endpoints. When a broken - server implementation is used, restic prints errors similar to the following: + Some S3 API implementations, e.g. Ceph before version 14.2.5, have a broken + `ListObjectsV2` implementation which causes problems for restic when using their + API endpoints. When a broken server implementation is used, restic prints errors + similar to the following: List() returned error: Truncated response should have continuation token set - As a temporary workaround, restic now allows using the older `ListObjects` endpoint by - setting the `s3.list-objects-v1` extended option, for instance: + As a temporary workaround, restic now allows using the older `ListObjects` + endpoint by setting the `s3.list-objects-v1` extended option, for instance: Restic -o s3.list-objects-v1=true snapshots @@ -2670,28 +2865,30 @@ restic users. The changes are ordered by importance. * Enhancement #3099: Reduce memory usage of `check` command - The `check` command now requires less memory if it is run without the `--check-unused` option. + The `check` command now requires less memory if it is run without the + `--check-unused` option. https://github.com/restic/restic/pull/3099 * Enhancement #3106: Parallelize scan of snapshot content in `copy` and `prune` - The `copy` and `prune` commands used to traverse the directories of snapshots one by one to find - used data. This snapshot traversal is now parallized which can speed up this step several - times. + The `copy` and `prune` commands used to traverse the directories of snapshots + one by one to find used data. This snapshot traversal is now parallized which + can speed up this step several times. - In addition the `check` command now reports how many snapshots have already been processed. + In addition the `check` command now reports how many snapshots have already been + processed. https://github.com/restic/restic/pull/3106 * Enhancement #3130: Parallelize reading of locks and snapshots - Restic used to read snapshots sequentially. For repositories containing many snapshots this - slowed down commands which have to read all snapshots. + Restic used to read snapshots sequentially. For repositories containing many + snapshots this slowed down commands which have to read all snapshots. - Now the reading of snapshots is parallelized. This speeds up for example `prune`, `backup` and - other commands that search for snapshots with certain properties or which have to find the - `latest` snapshot. + Now the reading of snapshots is parallelized. This speeds up for example + `prune`, `backup` and other commands that search for snapshots with certain + properties or which have to find the `latest` snapshot. The speed up also applies to locks stored in the backup repository. @@ -2700,37 +2897,39 @@ restic users. The changes are ordered by importance. * Enhancement #3147: Support additional environment variables for Swift authentication - The `swift` backend now supports the following additional environment variables for passing - authentication details to restic: `OS_USER_ID`, `OS_USER_DOMAIN_ID`, + The `swift` backend now supports the following additional environment variables + for passing authentication details to restic: `OS_USER_ID`, `OS_USER_DOMAIN_ID`, `OS_PROJECT_DOMAIN_ID` and `OS_TRUST_ID` - Depending on the `openrc` configuration file these might be required when the user and project - domains differ from one another. + Depending on the `openrc` configuration file these might be required when the + user and project domains differ from one another. https://github.com/restic/restic/issues/3147 https://github.com/restic/restic/pull/3158 * Enhancement #3191: Add release binaries for MIPS architectures - We've added a few new architectures for Linux to the release binaries: `mips`, `mipsle`, - `mips64`, and `mip64le`. MIPS is mostly used for low-end embedded systems. + We've added a few new architectures for Linux to the release binaries: `mips`, + `mipsle`, `mips64`, and `mip64le`. MIPS is mostly used for low-end embedded + systems. https://github.com/restic/restic/issues/3191 https://github.com/restic/restic/pull/3208 * Enhancement #3250: Add several more error checks - We've added a lot more error checks in places where errors were previously ignored (as hinted by - the static analysis program `errcheck` via `golangci-lint`). + We've added a lot more error checks in places where errors were previously + ignored (as hinted by the static analysis program `errcheck` via + `golangci-lint`). https://github.com/restic/restic/pull/3250 * Enhancement #3254: Enable HTTP/2 for backend connections - Go's HTTP library usually automatically chooses between HTTP/1.x and HTTP/2 depending on - what the server supports. But for compatibility this mechanism is disabled if DialContext is - used (which is the case for restic). This change allows restic's HTTP client to negotiate - HTTP/2 if supported by the server. + Go's HTTP library usually automatically chooses between HTTP/1.x and HTTP/2 + depending on what the server supports. But for compatibility this mechanism is + disabled if DialContext is used (which is the case for restic). This change + allows restic's HTTP client to negotiate HTTP/2 if supported by the server. https://github.com/restic/restic/pull/3254 @@ -2761,11 +2960,11 @@ restic users. The changes are ordered by importance. * Bugfix #1212: Restore timestamps and permissions on intermediate directories - When using the `--include` option of the restore command, restic restored timestamps and - permissions only on directories selected by the include pattern. Intermediate directories, - which are necessary to restore files located in sub- directories, were created with default - permissions. We've fixed the restore command to restore timestamps and permissions for these - directories as well. + When using the `--include` option of the restore command, restic restored + timestamps and permissions only on directories selected by the include pattern. + Intermediate directories, which are necessary to restore files located in sub- + directories, were created with default permissions. We've fixed the restore + command to restore timestamps and permissions for these directories as well. https://github.com/restic/restic/issues/1212 https://github.com/restic/restic/issues/1402 @@ -2773,13 +2972,15 @@ restic users. The changes are ordered by importance. * Bugfix #1756: Mark repository files as read-only when using the local backend - Files stored in a local repository were marked as writeable on the filesystem for non-Windows - systems, which did not prevent accidental file modifications outside of restic. In addition, - the local backend did not work with certain filesystems and network mounts which do not permit - modifications of file permissions. + Files stored in a local repository were marked as writeable on the filesystem + for non-Windows systems, which did not prevent accidental file modifications + outside of restic. In addition, the local backend did not work with certain + filesystems and network mounts which do not permit modifications of file + permissions. - Restic now marks files stored in a local repository as read-only on the filesystem on - non-Windows systems. The error handling is improved to support more filesystems. + Restic now marks files stored in a local repository as read-only on the + filesystem on non-Windows systems. The error handling is improved to support + more filesystems. https://github.com/restic/restic/issues/1756 https://github.com/restic/restic/issues/2157 @@ -2787,8 +2988,9 @@ restic users. The changes are ordered by importance. * Bugfix #2241: Hide password in REST backend repository URLs - When using a password in the REST backend repository URL, the password could in some cases be - included in the output from restic, e.g. when initializing a repo or during an error. + When using a password in the REST backend repository URL, the password could in + some cases be included in the output from restic, e.g. when initializing a repo + or during an error. The password is now replaced with "***" where applicable. @@ -2797,10 +2999,11 @@ restic users. The changes are ordered by importance. * Bugfix #2319: Correctly dump directories into tar files - The dump command previously wrote directories in a tar file in a way which can cause - compatibility problems. This caused, for example, 7zip on Windows to not open tar files - containing directories. In addition it was not possible to dump directories with extended - attributes. These compatibility problems are now corrected. + The dump command previously wrote directories in a tar file in a way which can + cause compatibility problems. This caused, for example, 7zip on Windows to not + open tar files containing directories. In addition it was not possible to dump + directories with extended attributes. These compatibility problems are now + corrected. In addition, a tar file now includes the name of the owner and group of a file. @@ -2809,17 +3012,18 @@ restic users. The changes are ordered by importance. * Bugfix #2491: Don't require `self-update --output` placeholder file - `restic self-update --output /path/to/new-restic` used to require that new-restic was an - existing file, to be overwritten. Now it's possible to download an updated restic binary to a - new path, without first having to create a placeholder file. + `restic self-update --output /path/to/new-restic` used to require that + new-restic was an existing file, to be overwritten. Now it's possible to + download an updated restic binary to a new path, without first having to create + a placeholder file. https://github.com/restic/restic/issues/2491 https://github.com/restic/restic/pull/2937 * Bugfix #2834: Fix rare cases of backup command hanging forever - We've fixed an issue with the backup progress reporting which could cause restic to hang - forever right before finishing a backup. + We've fixed an issue with the backup progress reporting which could cause restic + to hang forever right before finishing a backup. https://github.com/restic/restic/issues/2834 https://github.com/restic/restic/pull/2963 @@ -2833,47 +3037,50 @@ restic users. The changes are ordered by importance. * Bugfix #2942: Make --exclude-larger-than handle disappearing files - There was a small bug in the backup command's --exclude-larger-than option where files that - disappeared between scanning and actually backing them up to the repository caused a panic. - This is now fixed. + There was a small bug in the backup command's --exclude-larger-than option where + files that disappeared between scanning and actually backing them up to the + repository caused a panic. This is now fixed. https://github.com/restic/restic/issues/2942 * Bugfix #2951: Restic generate, help and self-update no longer check passwords - The commands `restic cache`, `generate`, `help` and `self-update` don't need passwords, but - they previously did run the RESTIC_PASSWORD_COMMAND (if set in the environment), prompting - users to authenticate for no reason. They now skip running the password command. + The commands `restic cache`, `generate`, `help` and `self-update` don't need + passwords, but they previously did run the RESTIC_PASSWORD_COMMAND (if set in + the environment), prompting users to authenticate for no reason. They now skip + running the password command. https://github.com/restic/restic/issues/2951 https://github.com/restic/restic/pull/2987 * Bugfix #2979: Make snapshots --json output [] instead of null when no snapshots - Restic previously output `null` instead of `[]` for the `--json snapshots` command, when - there were no snapshots in the repository. This caused some minor problems when parsing the - output, but is now fixed such that `[]` is output when the list of snapshots is empty. + Restic previously output `null` instead of `[]` for the `--json snapshots` + command, when there were no snapshots in the repository. This caused some minor + problems when parsing the output, but is now fixed such that `[]` is output when + the list of snapshots is empty. https://github.com/restic/restic/issues/2979 https://github.com/restic/restic/pull/2984 * Enhancement #340: Add support for Volume Shadow Copy Service (VSS) on Windows - Volume Shadow Copy Service allows read access to files that are locked by another process using - an exclusive lock through a filesystem snapshot. Restic was unable to backup those files - before. This update enables backing up these files. + Volume Shadow Copy Service allows read access to files that are locked by + another process using an exclusive lock through a filesystem snapshot. Restic + was unable to backup those files before. This update enables backing up these + files. - This needs to be enabled explicitely using the --use-fs-snapshot option of the backup - command. + This needs to be enabled explicitely using the --use-fs-snapshot option of the + backup command. https://github.com/restic/restic/issues/340 https://github.com/restic/restic/pull/2274 * Enhancement #1458: New option --repository-file - We've added a new command-line option --repository-file as an alternative to -r. This allows - to read the repository URL from a file in order to prevent certain types of information leaks, - especially for URLs containing credentials. + We've added a new command-line option --repository-file as an alternative to -r. + This allows to read the repository URL from a file in order to prevent certain + types of information leaks, especially for URLs containing credentials. https://github.com/restic/restic/issues/1458 https://github.com/restic/restic/issues/2900 @@ -2881,27 +3088,29 @@ restic users. The changes are ordered by importance. * Enhancement #2849: Authenticate to Google Cloud Storage with access token - When using the GCS backend, it is now possible to authenticate with OAuth2 access tokens - instead of a credentials file by setting the GOOGLE_ACCESS_TOKEN environment variable. + When using the GCS backend, it is now possible to authenticate with OAuth2 + access tokens instead of a credentials file by setting the GOOGLE_ACCESS_TOKEN + environment variable. https://github.com/restic/restic/pull/2849 * Enhancement #2969: Optimize check for unchanged files during backup - During a backup restic skips processing files which have not changed since the last backup run. - Previously this required opening each file once which can be slow on network filesystems. The - backup command now checks for file changes before opening a file. This considerably reduces - the time to create a backup on network filesystems. + During a backup restic skips processing files which have not changed since the + last backup run. Previously this required opening each file once which can be + slow on network filesystems. The backup command now checks for file changes + before opening a file. This considerably reduces the time to create a backup on + network filesystems. https://github.com/restic/restic/issues/2969 https://github.com/restic/restic/pull/2970 * Enhancement #2978: Warn if parent snapshot cannot be loaded during backup - During a backup restic uses the parent snapshot to check whether a file was changed and has to be - backed up again. For this check the backup has to read the directories contained in the old - snapshot. If a tree blob cannot be loaded, restic now warns about this problem with the backup - repository. + During a backup restic uses the parent snapshot to check whether a file was + changed and has to be backed up again. For this check the backup has to read the + directories contained in the old snapshot. If a tree blob cannot be loaded, + restic now warns about this problem with the backup repository. https://github.com/restic/restic/pull/2978 @@ -2961,15 +3170,16 @@ restic users. The changes are ordered by importance. * Bugfix #1863: Report correct number of directories processed by backup - The directory statistics calculation was fixed to report the actual number of processed - directories instead of always zero. + The directory statistics calculation was fixed to report the actual number of + processed directories instead of always zero. https://github.com/restic/restic/issues/1863 * Bugfix #2254: Fix tar issues when dumping `/` - We've fixed an issue with dumping either `/` or files on the first sublevel e.g. `/foo` to tar. - This also fixes tar dumping issues on Windows where this issue could also happen. + We've fixed an issue with dumping either `/` or files on the first sublevel e.g. + `/foo` to tar. This also fixes tar dumping issues on Windows where this issue + could also happen. https://github.com/restic/restic/issues/2254 https://github.com/restic/restic/issues/2357 @@ -2977,59 +3187,63 @@ restic users. The changes are ordered by importance. * Bugfix #2281: Handle format verbs like '%' properly in `find` output - The JSON or "normal" output of the `find` command can now deal with file names that contain - substrings which the Golang `fmt` package considers "format verbs" like `%s`. + The JSON or "normal" output of the `find` command can now deal with file names + that contain substrings which the Golang `fmt` package considers "format verbs" + like `%s`. https://github.com/restic/restic/issues/2281 * Bugfix #2298: Do not hang when run as a background job - Restic did hang on exit while restoring the terminal configuration when it was started as a - background job, for example using `restic ... &`. This has been fixed by only restoring the - terminal configuration when restic is interrupted while reading a password from the - terminal. + Restic did hang on exit while restoring the terminal configuration when it was + started as a background job, for example using `restic ... &`. This has been + fixed by only restoring the terminal configuration when restic is interrupted + while reading a password from the terminal. https://github.com/restic/restic/issues/2298 * Bugfix #2389: Fix mangled json output of backup command - We've fixed a race condition in the json output of the backup command that could cause multiple - lines to get mixed up. We've also ensured that the backup summary is printed last. + We've fixed a race condition in the json output of the backup command that could + cause multiple lines to get mixed up. We've also ensured that the backup summary + is printed last. https://github.com/restic/restic/issues/2389 https://github.com/restic/restic/pull/2545 * Bugfix #2390: Refresh lock timestamp - Long-running operations did not refresh lock timestamp, resulting in locks becoming stale. - This is now fixed. + Long-running operations did not refresh lock timestamp, resulting in locks + becoming stale. This is now fixed. https://github.com/restic/restic/issues/2390 * Bugfix #2429: Backup --json reports total_bytes_processed as 0 - We've fixed the json output of total_bytes_processed. The non-json output was already fixed - with pull request #2138 but left the json output untouched. + We've fixed the json output of total_bytes_processed. The non-json output was + already fixed with pull request #2138 but left the json output untouched. https://github.com/restic/restic/issues/2429 * Bugfix #2469: Fix incorrect bytes stats in `diff` command - In some cases, the wrong number of bytes (e.g. 16777215.998 TiB) were reported by the `diff` - command. This is now fixed. + In some cases, the wrong number of bytes (e.g. 16777215.998 TiB) were reported + by the `diff` command. This is now fixed. https://github.com/restic/restic/issues/2469 * Bugfix #2518: Do not crash with Synology NAS sftp server - It was found that when restic is used to store data on an sftp server on a Synology NAS with a - relative path (one which does not start with a slash), it may go into an endless loop trying to - create directories on the server. We've fixed this bug by using a function in the sftp library - instead of our own implementation. + It was found that when restic is used to store data on an sftp server on a + Synology NAS with a relative path (one which does not start with a slash), it + may go into an endless loop trying to create directories on the server. We've + fixed this bug by using a function in the sftp library instead of our own + implementation. - The bug was discovered because the Synology sftp server behaves erratic with non-absolute - path (e.g. `home/restic-repo`). This can be resolved by just using an absolute path instead - (`/home/restic-repo`). We've also added a paragraph in the FAQ. + The bug was discovered because the Synology sftp server behaves erratic with + non-absolute path (e.g. `home/restic-repo`). This can be resolved by just using + an absolute path instead (`/home/restic-repo`). We've also added a paragraph in + the FAQ. https://github.com/restic/restic/issues/2518 https://github.com/restic/restic/issues/2363 @@ -3037,84 +3251,90 @@ restic users. The changes are ordered by importance. * Bugfix #2531: Fix incorrect size calculation in `stats --mode restore-size` - The restore-size mode of stats was counting hard-linked files as if they were independent. + The restore-size mode of stats was counting hard-linked files as if they were + independent. https://github.com/restic/restic/issues/2531 * Bugfix #2537: Fix incorrect file counts in `stats --mode restore-size` - The restore-size mode of stats was failing to count empty directories and some files with hard - links. + The restore-size mode of stats was failing to count empty directories and some + files with hard links. https://github.com/restic/restic/issues/2537 * Bugfix #2592: SFTP backend supports IPv6 addresses - The SFTP backend now supports IPv6 addresses natively, without relying on aliases in the - external SSH configuration. + The SFTP backend now supports IPv6 addresses natively, without relying on + aliases in the external SSH configuration. https://github.com/restic/restic/pull/2592 * Bugfix #2607: Honor RESTIC_CACHE_DIR environment variable on Mac and Windows - On Mac and Windows, the RESTIC_CACHE_DIR environment variable was ignored. This variable can - now be used on all platforms to set the directory where restic stores caches. + On Mac and Windows, the RESTIC_CACHE_DIR environment variable was ignored. This + variable can now be used on all platforms to set the directory where restic + stores caches. https://github.com/restic/restic/pull/2607 * Bugfix #2668: Don't abort the stats command when data blobs are missing - Runing the stats command in the blobs-per-file mode on a repository with missing data blobs - previously resulted in a crash. + Runing the stats command in the blobs-per-file mode on a repository with missing + data blobs previously resulted in a crash. https://github.com/restic/restic/pull/2668 * Bugfix #2674: Add stricter prune error checks - Additional checks were added to the prune command in order to improve resiliency to backend, - hardware and/or networking issues. The checks now detect a few more cases where such outside - factors could potentially cause data loss. + Additional checks were added to the prune command in order to improve resiliency + to backend, hardware and/or networking issues. The checks now detect a few more + cases where such outside factors could potentially cause data loss. https://github.com/restic/restic/pull/2674 * Bugfix #2899: Fix possible crash in the progress bar of check --read-data - We've fixed a possible crash while displaying the progress bar for the check --read-data - command. The crash occurred when the length of the progress bar status exceeded the terminal - width, which only happened for very narrow terminal windows. + We've fixed a possible crash while displaying the progress bar for the check + --read-data command. The crash occurred when the length of the progress bar + status exceeded the terminal width, which only happened for very narrow terminal + windows. https://github.com/restic/restic/pull/2899 https://forum.restic.net/t/restic-rclone-pcloud-connection-issues/2963/15 * Change #1597: Honor the --no-lock flag in the mount command - The mount command now does not lock the repository if given the --no-lock flag. This allows to - mount repositories which are archived on a read only backend/filesystem. + The mount command now does not lock the repository if given the --no-lock flag. + This allows to mount repositories which are archived on a read only + backend/filesystem. https://github.com/restic/restic/issues/1597 https://github.com/restic/restic/pull/2821 * Change #2482: Remove vendored dependencies - We've removed the vendored dependencies (in the subdir `vendor/`). When building restic, the - Go compiler automatically fetches the dependencies. It will also cryptographically verify - that the correct code has been fetched by using the hashes in `go.sum` (see the link to the - documentation below). + We've removed the vendored dependencies (in the subdir `vendor/`). When building + restic, the Go compiler automatically fetches the dependencies. It will also + cryptographically verify that the correct code has been fetched by using the + hashes in `go.sum` (see the link to the documentation below). https://github.com/restic/restic/issues/2482 https://golang.org/cmd/go/#hdr-Module_downloading_and_verification * Change #2546: Return exit code 3 when failing to backup all source data - The backup command used to return a zero exit code as long as a snapshot could be created - successfully, even if some of the source files could not be read (in which case the snapshot - would contain the rest of the files). + The backup command used to return a zero exit code as long as a snapshot could + be created successfully, even if some of the source files could not be read (in + which case the snapshot would contain the rest of the files). - This made it hard for automation/scripts to detect failures/incomplete backups by looking at - the exit code. Restic now returns the following exit codes for the backup command: + This made it hard for automation/scripts to detect failures/incomplete backups + by looking at the exit code. Restic now returns the following exit codes for the + backup command: - - 0 when the command was successful - 1 when there was a fatal error (no snapshot created) - 3 when - some source data could not be read (incomplete snapshot created) + - 0 when the command was successful - 1 when there was a fatal error (no + snapshot created) - 3 when some source data could not be read (incomplete + snapshot created) https://github.com/restic/restic/issues/956 https://github.com/restic/restic/issues/2064 @@ -3124,12 +3344,12 @@ restic users. The changes are ordered by importance. * Change #2600: Update dependencies, require Go >= 1.13 - Restic now requires Go to be at least 1.13. This allows simplifications in the build process and - removing workarounds. + Restic now requires Go to be at least 1.13. This allows simplifications in the + build process and removing workarounds. - This is also probably the last version of restic still supporting mounting repositories via - fuse on macOS. The library we're using for fuse does not support macOS any more and osxfuse is not - open source any more. + This is also probably the last version of restic still supporting mounting + repositories via fuse on macOS. The library we're using for fuse does not + support macOS any more and osxfuse is not open source any more. https://github.com/bazil/fuse/issues/224 https://github.com/osxfuse/osxfuse/issues/590 @@ -3139,17 +3359,20 @@ restic users. The changes are ordered by importance. * Enhancement #323: Add command for copying snapshots between repositories - We've added a copy command, allowing you to copy snapshots from one repository to another. + We've added a copy command, allowing you to copy snapshots from one repository + to another. - Note that this process will have to read (download) and write (upload) the entire snapshot(s) - due to the different encryption keys used on the source and destination repository. Also, the - transferred files are not re-chunked, which may break deduplication between files already - stored in the destination repo and files copied there using this command. + Note that this process will have to read (download) and write (upload) the + entire snapshot(s) due to the different encryption keys used on the source and + destination repository. Also, the transferred files are not re-chunked, which + may break deduplication between files already stored in the destination repo and + files copied there using this command. - To fully support deduplication between repositories when the copy command is used, the init - command now supports the `--copy-chunker-params` option, which initializes the new - repository with identical parameters for splitting files into chunks as an already existing - repository. This allows copied snapshots to be equally deduplicated in both repositories. + To fully support deduplication between repositories when the copy command is + used, the init command now supports the `--copy-chunker-params` option, which + initializes the new repository with identical parameters for splitting files + into chunks as an already existing repository. This allows copied snapshots to + be equally deduplicated in both repositories. https://github.com/restic/restic/issues/323 https://github.com/restic/restic/pull/2606 @@ -3157,29 +3380,29 @@ restic users. The changes are ordered by importance. * Enhancement #551: Use optimized library for hash calculation of file chunks - We've switched the library used to calculate the hashes of file chunks, which are used for - deduplication, to the optimized Minio SHA-256 implementation. + We've switched the library used to calculate the hashes of file chunks, which + are used for deduplication, to the optimized Minio SHA-256 implementation. - Depending on the CPU it improves the hashing throughput by 10-30%. Modern x86 CPUs with the SHA - Extension should be about two to three times faster. + Depending on the CPU it improves the hashing throughput by 10-30%. Modern x86 + CPUs with the SHA Extension should be about two to three times faster. https://github.com/restic/restic/issues/551 https://github.com/restic/restic/pull/2709 * Enhancement #1570: Support specifying multiple host flags for various commands - Previously commands didn't take more than one `--host` or `-H` argument into account, which - could be limiting with e.g. the `forget` command. + Previously commands didn't take more than one `--host` or `-H` argument into + account, which could be limiting with e.g. the `forget` command. - The `dump`, `find`, `forget`, `ls`, `mount`, `restore`, `snapshots`, `stats` and `tag` - commands will now take into account multiple `--host` and `-H` flags. + The `dump`, `find`, `forget`, `ls`, `mount`, `restore`, `snapshots`, `stats` and + `tag` commands will now take into account multiple `--host` and `-H` flags. https://github.com/restic/restic/issues/1570 * Enhancement #1680: Optimize `restic mount` - We've optimized the FUSE implementation used within restic. `restic mount` is now more - responsive and uses less memory. + We've optimized the FUSE implementation used within restic. `restic mount` is + now more responsive and uses less memory. https://github.com/restic/restic/issues/1680 https://github.com/restic/restic/pull/2587 @@ -3193,10 +3416,11 @@ restic users. The changes are ordered by importance. * Enhancement #2175: Allow specifying user and host when creating keys - When adding a new key to the repository, the username and hostname for the new key can be - specified on the command line. This allows overriding the defaults, for example if you would - prefer to use the FQDN to identify the host or if you want to add keys for several different hosts - without having to run the key add command on those hosts. + When adding a new key to the repository, the username and hostname for the new + key can be specified on the command line. This allows overriding the defaults, + for example if you would prefer to use the FQDN to identify the host or if you + want to add keys for several different hosts without having to run the key add + command on those hosts. https://github.com/restic/restic/issues/2175 @@ -3210,15 +3434,16 @@ restic users. The changes are ordered by importance. Fixes "not enough cache capacity" error during restore: https://github.com/restic/restic/issues/2244 - NOTE: This new implementation does not guarantee order in which blobs are written to the target - files and, for example, the last blob of a file can be written to the file before any of the - preceeding file blobs. It is therefore possible to have gaps in the data written to the target - files if restore fails or interrupted by the user. + NOTE: This new implementation does not guarantee order in which blobs are + written to the target files and, for example, the last blob of a file can be + written to the file before any of the preceeding file blobs. It is therefore + possible to have gaps in the data written to the target files if restore fails + or interrupted by the user. - The implementation will try to preallocate space for the restored files on the filesystem to - prevent file fragmentation. This ensures good read performance for large files, like for - example VM images. If preallocating space is not supported by the filesystem, then this step is - silently skipped. + The implementation will try to preallocate space for the restored files on the + filesystem to prevent file fragmentation. This ensures good read performance for + large files, like for example VM images. If preallocating space is not supported + by the filesystem, then this step is silently skipped. https://github.com/restic/restic/pull/2195 https://github.com/restic/restic/pull/2893 @@ -3231,69 +3456,73 @@ restic users. The changes are ordered by importance. * Enhancement #2328: Improve speed of check command - We've improved the check command to traverse trees only once independent of whether they are - contained in multiple snapshots. The check command is now much faster for repositories with a - large number of snapshots. + We've improved the check command to traverse trees only once independent of + whether they are contained in multiple snapshots. The check command is now much + faster for repositories with a large number of snapshots. https://github.com/restic/restic/issues/2284 https://github.com/restic/restic/pull/2328 * Enhancement #2395: Ignore sync errors when operation not supported by local filesystem - The local backend has been modified to work with filesystems which doesn't support the `sync` - operation. This operation is normally used by restic to ensure that data files are fully - written to disk before continuing. + The local backend has been modified to work with filesystems which doesn't + support the `sync` operation. This operation is normally used by restic to + ensure that data files are fully written to disk before continuing. - For these limited filesystems, saving a file in the backend would previously fail with an - "operation not supported" error. This error is now ignored, which means that e.g. an SMB mount - on macOS can now be used as storage location for a repository. + For these limited filesystems, saving a file in the backend would previously + fail with an "operation not supported" error. This error is now ignored, which + means that e.g. an SMB mount on macOS can now be used as storage location for a + repository. https://github.com/restic/restic/issues/2395 https://forum.restic.net/t/sync-errors-on-mac-over-smb/1859 * Enhancement #2423: Support user@domain parsing as user - Added the ability for user@domain-like users to be authenticated over SFTP servers. + Added the ability for user@domain-like users to be authenticated over SFTP + servers. https://github.com/restic/restic/pull/2423 * Enhancement #2427: Add flag `--iexclude-file` to backup command - The backup command now supports the flag `--iexclude-file` which is a case-insensitive - version of `--exclude-file`. + The backup command now supports the flag `--iexclude-file` which is a + case-insensitive version of `--exclude-file`. https://github.com/restic/restic/issues/2427 https://github.com/restic/restic/pull/2898 * Enhancement #2569: Support excluding files by their size - The `backup` command now supports the `--exclude-larger-than` option to exclude files which - are larger than the specified maximum size. This can for example be useful to exclude - unimportant files with a large file size. + The `backup` command now supports the `--exclude-larger-than` option to exclude + files which are larger than the specified maximum size. This can for example be + useful to exclude unimportant files with a large file size. https://github.com/restic/restic/issues/2569 https://github.com/restic/restic/pull/2914 * Enhancement #2571: Self-heal missing file parts during backup of unchanged files - We've improved the resilience of restic to certain types of repository corruption. + We've improved the resilience of restic to certain types of repository + corruption. - For files that are unchanged since the parent snapshot, the backup command now verifies that - all parts of the files still exist in the repository. Parts that are missing, e.g. from a damaged - repository, are backed up again. This verification was already run for files that were - modified since the parent snapshot, but is now also done for unchanged files. + For files that are unchanged since the parent snapshot, the backup command now + verifies that all parts of the files still exist in the repository. Parts that + are missing, e.g. from a damaged repository, are backed up again. This + verification was already run for files that were modified since the parent + snapshot, but is now also done for unchanged files. - Note that restic will not backup file parts that are referenced in the index but where the actual - data is not present on disk, as this situation can only be detected by restic check. Please - ensure that you run `restic check` regularly. + Note that restic will not backup file parts that are referenced in the index but + where the actual data is not present on disk, as this situation can only be + detected by restic check. Please ensure that you run `restic check` regularly. https://github.com/restic/restic/issues/2571 https://github.com/restic/restic/pull/2827 * Enhancement #2576: Improve the chunking algorithm - We've updated the chunker library responsible for splitting files into smaller blocks. It - should improve the chunking throughput by 5-15% depending on the CPU. + We've updated the chunker library responsible for splitting files into smaller + blocks. It should improve the chunking throughput by 5-15% depending on the CPU. https://github.com/restic/restic/issues/2820 https://github.com/restic/restic/pull/2576 @@ -3301,65 +3530,68 @@ restic users. The changes are ordered by importance. * Enhancement #2598: Improve speed of diff command - We've improved the performance of the diff command when comparing snapshots with similar - content. It should run up to twice as fast as before. + We've improved the performance of the diff command when comparing snapshots with + similar content. It should run up to twice as fast as before. https://github.com/restic/restic/pull/2598 * Enhancement #2599: Slightly reduce memory usage of prune and stats commands - The prune and the stats command kept directory identifiers in memory twice while searching for - used blobs. + The prune and the stats command kept directory identifiers in memory twice while + searching for used blobs. https://github.com/restic/restic/pull/2599 * Enhancement #2733: S3 backend: Add support for WebIdentityTokenFile - We've added support for EKS IAM roles for service accounts feature to the S3 backend. + We've added support for EKS IAM roles for service accounts feature to the S3 + backend. https://github.com/restic/restic/issues/2703 https://github.com/restic/restic/pull/2733 * Enhancement #2773: Optimize handling of new index entries - Restic now uses less memory for backups which add a lot of data, e.g. large initial backups. In - addition, we've improved the stability in some edge cases. + Restic now uses less memory for backups which add a lot of data, e.g. large + initial backups. In addition, we've improved the stability in some edge cases. https://github.com/restic/restic/pull/2773 * Enhancement #2781: Reduce memory consumption of in-memory index - We've improved how the index is stored in memory. This change can reduce memory usage for large - repositories by up to 50% (depending on the operation). + We've improved how the index is stored in memory. This change can reduce memory + usage for large repositories by up to 50% (depending on the operation). https://github.com/restic/restic/pull/2781 https://github.com/restic/restic/pull/2812 * Enhancement #2786: Optimize `list blobs` command - We've changed the implementation of `list blobs` which should be now a bit faster and consume - almost no memory even for large repositories. + We've changed the implementation of `list blobs` which should be now a bit + faster and consume almost no memory even for large repositories. https://github.com/restic/restic/pull/2786 * Enhancement #2790: Optimized file access in restic mount - Reading large (> 100GiB) files from restic mountpoints is now faster, and the speedup is - greater for larger files. + Reading large (> 100GiB) files from restic mountpoints is now faster, and the + speedup is greater for larger files. https://github.com/restic/restic/pull/2790 * Enhancement #2840: Speed-up file deletion in forget, prune and rebuild-index - We've sped up the file deletion for the commands forget, prune and rebuild-index, especially - for remote repositories. Deletion was sequential before and is now run in parallel. + We've sped up the file deletion for the commands forget, prune and + rebuild-index, especially for remote repositories. Deletion was sequential + before and is now run in parallel. https://github.com/restic/restic/pull/2840 * Enhancement #2858: Support filtering snapshots by tag and path in the stats command - We've added filtering snapshots by `--tag tagList` and by `--path path` to the `stats` - command. This includes filtering of only 'latest' snapshots or all snapshots in a repository. + We've added filtering snapshots by `--tag tagList` and by `--path path` to the + `stats` command. This includes filtering of only 'latest' snapshots or all + snapshots in a repository. https://github.com/restic/restic/issues/2858 https://github.com/restic/restic/pull/2859 @@ -3386,81 +3618,85 @@ restic users. The changes are ordered by importance. * Bugfix #2063: Allow absolute path for filename when backing up from stdin - When backing up from stdin, handle directory path for `--stdin-filename`. This can be used to - specify the full path for the backed-up file. + When backing up from stdin, handle directory path for `--stdin-filename`. This + can be used to specify the full path for the backed-up file. https://github.com/restic/restic/issues/2063 * Bugfix #2174: Save files with invalid timestamps - When restic reads invalid timestamps (year is before 0000 or after 9999) it refused to read and - archive the file. We've changed the behavior and will now save modified timestamps with the - year set to either 0000 or 9999, the rest of the timestamp stays the same, so the file will be saved - (albeit with a bogus timestamp). + When restic reads invalid timestamps (year is before 0000 or after 9999) it + refused to read and archive the file. We've changed the behavior and will now + save modified timestamps with the year set to either 0000 or 9999, the rest of + the timestamp stays the same, so the file will be saved (albeit with a bogus + timestamp). https://github.com/restic/restic/issues/2174 https://github.com/restic/restic/issues/1173 * Bugfix #2249: Read fresh metadata for unmodified files - Restic took all metadata for files which were detected as unmodified, not taking into account - changed metadata (ownership, mode). This is now corrected. + Restic took all metadata for files which were detected as unmodified, not taking + into account changed metadata (ownership, mode). This is now corrected. https://github.com/restic/restic/issues/2249 https://github.com/restic/restic/pull/2252 * Bugfix #2301: Add upper bound for t in --read-data-subset=n/t - 256 is the effective maximum for t, but restic would allow larger values, leading to strange - behavior. + 256 is the effective maximum for t, but restic would allow larger values, + leading to strange behavior. https://github.com/restic/restic/issues/2301 https://github.com/restic/restic/pull/2304 * Bugfix #2321: Check errors when loading index files - Restic now checks and handles errors which occur when loading index files, the missing check - leads to odd errors (and a stack trace printed to users) later. This was reported in the forum. + Restic now checks and handles errors which occur when loading index files, the + missing check leads to odd errors (and a stack trace printed to users) later. + This was reported in the forum. https://github.com/restic/restic/pull/2321 https://forum.restic.net/t/check-rebuild-index-prune/1848/13 * Enhancement #2179: Use ctime when checking for file changes - Previously, restic only checked a file's mtime (along with other non-timestamp metadata) to - decide if a file has changed. This could cause restic to not notice that a file has changed (and - therefore continue to store the old version, as opposed to the modified version) if something - edits the file and then resets the timestamp. Restic now also checks the ctime of files, so any - modifications to a file should be noticed, and the modified file will be backed up. The ctime - check will be disabled if the --ignore-inode flag was given. + Previously, restic only checked a file's mtime (along with other non-timestamp + metadata) to decide if a file has changed. This could cause restic to not notice + that a file has changed (and therefore continue to store the old version, as + opposed to the modified version) if something edits the file and then resets the + timestamp. Restic now also checks the ctime of files, so any modifications to a + file should be noticed, and the modified file will be backed up. The ctime check + will be disabled if the --ignore-inode flag was given. - If this change causes problems for you, please open an issue, and we can look in to adding a - seperate flag to disable just the ctime check. + If this change causes problems for you, please open an issue, and we can look in + to adding a seperate flag to disable just the ctime check. https://github.com/restic/restic/issues/2179 https://github.com/restic/restic/pull/2212 * Enhancement #2306: Allow multiple retries for interactive password input - Restic used to quit if the repository password was typed incorrectly once. Restic will now ask - the user again for the repository password if typed incorrectly. The user will now get three - tries to input the correct password before restic quits. + Restic used to quit if the repository password was typed incorrectly once. + Restic will now ask the user again for the repository password if typed + incorrectly. The user will now get three tries to input the correct password + before restic quits. https://github.com/restic/restic/issues/2306 * Enhancement #2330: Make `--group-by` accept both singular and plural - One can now use the values `host`/`hosts`, `path`/`paths` and `tag` / `tags` interchangeably - in the `--group-by` argument. + One can now use the values `host`/`hosts`, `path`/`paths` and `tag` / `tags` + interchangeably in the `--group-by` argument. https://github.com/restic/restic/issues/2330 * Enhancement #2350: Add option to configure S3 region - We've added a new option for setting the region when accessing an S3-compatible service. For - some providers, it is required to set this to a valid value. You can do that either by setting the - environment variable `AWS_DEFAULT_REGION` or using the option `s3.region`, e.g. like this: - `-o s3.region="us-east-1"`. + We've added a new option for setting the region when accessing an S3-compatible + service. For some providers, it is required to set this to a valid value. You + can do that either by setting the environment variable `AWS_DEFAULT_REGION` or + using the option `s3.region`, e.g. like this: `-o s3.region="us-east-1"`. https://github.com/restic/restic/pull/2350 @@ -3489,10 +3725,11 @@ restic users. The changes are ordered by importance. * Bugfix #2135: Return error when no bytes could be read from stdin - We assume that users reading backup data from stdin want to know when no data could be read, so now - restic returns an error when `backup --stdin` is called but no bytes could be read. Usually, - this means that an earlier command in a pipe has failed. The documentation was amended and now - recommends setting the `pipefail` option (`set -o pipefail`). + We assume that users reading backup data from stdin want to know when no data + could be read, so now restic returns an error when `backup --stdin` is called + but no bytes could be read. Usually, this means that an earlier command in a + pipe has failed. The documentation was amended and now recommends setting the + `pipefail` option (`set -o pipefail`). https://github.com/restic/restic/pull/2135 https://github.com/restic/restic/pull/2139 @@ -3503,84 +3740,88 @@ restic users. The changes are ordered by importance. * Bugfix #2203: Fix reading passwords from stdin - Passwords for the `init`, `key add`, and `key passwd` commands can now be read from - non-terminal stdin. + Passwords for the `init`, `key add`, and `key passwd` commands can now be read + from non-terminal stdin. https://github.com/restic/restic/issues/2203 * Bugfix #2224: Don't abort the find command when a tree can't be loaded - Change the find command so that missing trees don't result in a crash. Instead, the error is - logged to the debug log, and the tree ID is displayed along with the snapshot it belongs to. This - makes it possible to recover repositories that are missing trees by forgetting the snapshots - they are used in. + Change the find command so that missing trees don't result in a crash. Instead, + the error is logged to the debug log, and the tree ID is displayed along with + the snapshot it belongs to. This makes it possible to recover repositories that + are missing trees by forgetting the snapshots they are used in. https://github.com/restic/restic/issues/2224 * Enhancement #1895: Add case insensitive include & exclude options - The backup and restore commands now have --iexclude and --iinclude flags as case insensitive - variants of --exclude and --include. + The backup and restore commands now have --iexclude and --iinclude flags as case + insensitive variants of --exclude and --include. https://github.com/restic/restic/issues/1895 https://github.com/restic/restic/pull/2032 * Enhancement #1937: Support streaming JSON output for backup - We've added support for getting machine-readable status output during backup, just pass the - flag `--json` for `restic backup` and restic will output a stream of JSON objects which contain - the current progress. + We've added support for getting machine-readable status output during backup, + just pass the flag `--json` for `restic backup` and restic will output a stream + of JSON objects which contain the current progress. https://github.com/restic/restic/issues/1937 https://github.com/restic/restic/pull/1944 * Enhancement #2037: Add group-by option to snapshots command - We have added an option to group the output of the snapshots command, similar to the output of the - forget command. The option has been called "--group-by" and accepts any combination of the - values "host", "paths" and "tags", separated by commas. Default behavior (not specifying - --group-by) has not been changed. We have added support of the grouping to the JSON output. + We have added an option to group the output of the snapshots command, similar to + the output of the forget command. The option has been called "--group-by" and + accepts any combination of the values "host", "paths" and "tags", separated by + commas. Default behavior (not specifying --group-by) has not been changed. We + have added support of the grouping to the JSON output. https://github.com/restic/restic/issues/2037 https://github.com/restic/restic/pull/2087 * Enhancement #2124: Ability to dump folders to tar via stdout - We've added the ability to dump whole folders to stdout via the `dump` command. Restic now - requires at least Go 1.10 due to a limitation of the standard library for Go <= 1.9. + We've added the ability to dump whole folders to stdout via the `dump` command. + Restic now requires at least Go 1.10 due to a limitation of the standard library + for Go <= 1.9. https://github.com/restic/restic/issues/2123 https://github.com/restic/restic/pull/2124 * Enhancement #2139: Return error if no bytes could be read for `backup --stdin` - When restic is used to backup the output of a program, like `mysqldump | restic backup --stdin`, - it now returns an error if no bytes could be read at all. This catches the failure case when - `mysqldump` failed for some reason and did not output any data to stdout. + When restic is used to backup the output of a program, like `mysqldump | restic + backup --stdin`, it now returns an error if no bytes could be read at all. This + catches the failure case when `mysqldump` failed for some reason and did not + output any data to stdout. https://github.com/restic/restic/pull/2139 * Enhancement #2155: Add Openstack application credential auth for Swift - Since Openstack Queens Identity (auth V3) service supports an application credential auth - method. It allows to create a technical account with the limited roles. This commit adds an - application credential authentication method for the Swift backend. + Since Openstack Queens Identity (auth V3) service supports an application + credential auth method. It allows to create a technical account with the limited + roles. This commit adds an application credential authentication method for the + Swift backend. https://github.com/restic/restic/issues/2155 * Enhancement #2184: Add --json support to forget command - The forget command now supports the --json argument, outputting the information about what is - (or would-be) kept and removed from the repository. + The forget command now supports the --json argument, outputting the information + about what is (or would-be) kept and removed from the repository. https://github.com/restic/restic/issues/2184 https://github.com/restic/restic/pull/2185 * Enhancement #2205: Add --ignore-inode option to backup cmd - This option handles backup of virtual filesystems that do not keep fixed inodes for files, like - Fuse-based, pCloud, etc. Ignoring inode changes allows to consider the file as unchanged if - last modification date and size are unchanged. + This option handles backup of virtual filesystems that do not keep fixed inodes + for files, like Fuse-based, pCloud, etc. Ignoring inode changes allows to + consider the file as unchanged if last modification date and size are unchanged. https://github.com/restic/restic/issues/1631 https://github.com/restic/restic/pull/2205 @@ -3588,16 +3829,17 @@ restic users. The changes are ordered by importance. * Enhancement #2220: Add config option to set S3 storage class - The `s3.storage-class` option can be passed to restic (using `-o`) to specify the storage - class to be used for S3 objects created by restic. + The `s3.storage-class` option can be passed to restic (using `-o`) to specify + the storage class to be used for S3 objects created by restic. - The storage class is passed as-is to S3, so it needs to be understood by the API. On AWS, it can be - one of `STANDARD`, `STANDARD_IA`, `ONEZONE_IA`, `INTELLIGENT_TIERING` and - `REDUCED_REDUNDANCY`. If unspecified, the default storage class is used (`STANDARD` on - AWS). + The storage class is passed as-is to S3, so it needs to be understood by the + API. On AWS, it can be one of `STANDARD`, `STANDARD_IA`, `ONEZONE_IA`, + `INTELLIGENT_TIERING` and `REDUCED_REDUNDANCY`. If unspecified, the default + storage class is used (`STANDARD` on AWS). - You can mix storage classes in the same bucket, and the setting isn't stored in the restic - repository, so be sure to specify it with each command that writes to S3. + You can mix storage classes in the same bucket, and the setting isn't stored in + the restic repository, so be sure to specify it with each command that writes to + S3. https://github.com/restic/restic/issues/706 https://github.com/restic/restic/pull/2220 @@ -3625,19 +3867,19 @@ restic users. The changes are ordered by importance. * Bugfix #1989: Google Cloud Storage: Respect bandwidth limit - The GCS backend did not respect the bandwidth limit configured, a previous commit - accidentally removed support for it. + The GCS backend did not respect the bandwidth limit configured, a previous + commit accidentally removed support for it. https://github.com/restic/restic/issues/1989 https://github.com/restic/restic/pull/2100 * Bugfix #2040: Add host name filter shorthand flag for `stats` command - The default value for `--host` flag was set to 'H' (the shorthand version of the flag), this - caused the lookup for the latest snapshot to fail. + The default value for `--host` flag was set to 'H' (the shorthand version of the + flag), this caused the lookup for the latest snapshot to fail. - Add shorthand flag `-H` for `--host` (with empty default so if these flags are not specified the - latest snapshot will not filter by host name). + Add shorthand flag `-H` for `--host` (with empty default so if these flags are + not specified the latest snapshot will not filter by host name). Also add shorthand `-H` for `backup` command. @@ -3645,17 +3887,17 @@ restic users. The changes are ordered by importance. * Bugfix #2068: Correctly return error loading data - In one case during `prune` and `check`, an error loading data from the backend is not returned - properly. This is now corrected. + In one case during `prune` and `check`, an error loading data from the backend + is not returned properly. This is now corrected. https://github.com/restic/restic/issues/1999#issuecomment-433737921 https://github.com/restic/restic/pull/2068 * Bugfix #2095: Consistently use local time for snapshots times - By default snapshots created with restic backup were set to local time, but when the --time flag - was used the provided timestamp was parsed as UTC. With this change all snapshots times are set - to local time. + By default snapshots created with restic backup were set to local time, but when + the --time flag was used the provided timestamp was parsed as UTC. With this + change all snapshots times are set to local time. https://github.com/restic/restic/pull/2095 @@ -3664,65 +3906,70 @@ restic users. The changes are ordered by importance. This change significantly improves restore performance, especially when using high-latency remote repositories like B2. - The implementation now uses several concurrent threads to download and process multiple - remote files concurrently. To further reduce restore time, each remote file is downloaded - using a single repository request. + The implementation now uses several concurrent threads to download and process + multiple remote files concurrently. To further reduce restore time, each remote + file is downloaded using a single repository request. https://github.com/restic/restic/issues/1605 https://github.com/restic/restic/pull/1719 * Enhancement #2017: Mount: Enforce FUSE Unix permissions with allow-other - The fuse mount (`restic mount`) now lets the kernel check the permissions of the files within - snapshots (this is done through the `DefaultPermissions` FUSE option) when the option - `--allow-other` is specified. + The fuse mount (`restic mount`) now lets the kernel check the permissions of the + files within snapshots (this is done through the `DefaultPermissions` FUSE + option) when the option `--allow-other` is specified. - To restore the old behavior, we've added the `--no-default-permissions` option. This allows - all users that have access to the mount point to access all files within the snapshots. + To restore the old behavior, we've added the `--no-default-permissions` option. + This allows all users that have access to the mount point to access all files + within the snapshots. https://github.com/restic/restic/pull/2017 * Enhancement #2070: Make all commands display timestamps in local time - Restic used to drop the timezone information from displayed timestamps, it now converts - timestamps to local time before printing them so the times can be easily compared to. + Restic used to drop the timezone information from displayed timestamps, it now + converts timestamps to local time before printing them so the times can be + easily compared to. https://github.com/restic/restic/pull/2070 * Enhancement #2085: Allow --files-from to be specified multiple times - Before, restic took only the last file specified with `--files-from` into account, this is now - corrected. + Before, restic took only the last file specified with `--files-from` into + account, this is now corrected. https://github.com/restic/restic/issues/2085 https://github.com/restic/restic/pull/2086 * Enhancement #2089: Increase granularity of the "keep within" retention policy - The `keep-within` option of the `forget` command now accepts time ranges with an hourly - granularity. For example, running `restic forget --keep-within 3d12h` will keep all the - snapshots made within three days and twelve hours from the time of the latest snapshot. + The `keep-within` option of the `forget` command now accepts time ranges with an + hourly granularity. For example, running `restic forget --keep-within 3d12h` + will keep all the snapshots made within three days and twelve hours from the + time of the latest snapshot. https://github.com/restic/restic/issues/2089 https://github.com/restic/restic/pull/2090 * Enhancement #2094: Run command to get password - We've added the `--password-command` option which allows specifying a command that restic - runs every time the password for the repository is needed, so it can be integrated with a - password manager or keyring. The option can also be set via the environment variable - `$RESTIC_PASSWORD_COMMAND`. + We've added the `--password-command` option which allows specifying a command + that restic runs every time the password for the repository is needed, so it can + be integrated with a password manager or keyring. The option can also be set via + the environment variable `$RESTIC_PASSWORD_COMMAND`. https://github.com/restic/restic/pull/2094 * Enhancement #2097: Add key hinting - Added a new option `--key-hint` and corresponding environment variable `RESTIC_KEY_HINT`. - The key hint is a key ID to try decrypting first, before other keys in the repository. + Added a new option `--key-hint` and corresponding environment variable + `RESTIC_KEY_HINT`. The key hint is a key ID to try decrypting first, before + other keys in the repository. - This change will benefit repositories with many keys; if the correct key hint is supplied then - restic only needs to check one key. If the key hint is incorrect (the key does not exist, or the - password is incorrect) then restic will check all keys, as usual. + This change will benefit repositories with many keys; if the correct key hint is + supplied then restic only needs to check one key. If the key hint is incorrect + (the key does not exist, or the password is incorrect) then restic will check + all keys, as usual. https://github.com/restic/restic/issues/2097 @@ -3752,29 +3999,31 @@ restic users. The changes are ordered by importance. * Bugfix #1935: Remove truncated files from cache - When a file in the local cache is truncated, and restic tries to access data beyond the end of the - (cached) file, it used to return an error "EOF". This is now fixed, such truncated files are - removed and the data is fetched directly from the backend. + When a file in the local cache is truncated, and restic tries to access data + beyond the end of the (cached) file, it used to return an error "EOF". This is + now fixed, such truncated files are removed and the data is fetched directly + from the backend. https://github.com/restic/restic/issues/1935 * Bugfix #1978: Do not return an error when the scanner is slower than backup - When restic makes a backup, there's a background task called "scanner" which collects - information on how many files and directories are to be saved, in order to display progress - information to the user. When the backup finishes faster than the scanner, it is aborted - because the result is not needed any more. This logic contained a bug, where quitting the - scanner process was treated as an error, and caused restic to print an unhelpful error message - ("context canceled"). + When restic makes a backup, there's a background task called "scanner" which + collects information on how many files and directories are to be saved, in order + to display progress information to the user. When the backup finishes faster + than the scanner, it is aborted because the result is not needed any more. This + logic contained a bug, where quitting the scanner process was treated as an + error, and caused restic to print an unhelpful error message ("context + canceled"). https://github.com/restic/restic/issues/1978 https://github.com/restic/restic/pull/1991 * Enhancement #1766: Restore: suppress lchown errors when not running as root - Like "cp" and "rsync" do, restic now only reports errors for changing the ownership of files - during restore if it is run as root, on non-Windows operating systems. On Windows, the error - is reported as usual. + Like "cp" and "rsync" do, restic now only reports errors for changing the + ownership of files during restore if it is run as root, on non-Windows + operating systems. On Windows, the error is reported as usual. https://github.com/restic/restic/issues/1766 @@ -3782,113 +4031,118 @@ restic users. The changes are ordered by importance. We've updated the `find` command to support multiple patterns. - `restic find` is now able to list the snapshots containing a specific tree or blob, or even the - snapshots that contain blobs belonging to a given pack. A list of IDs can be given, as long as they - all have the same type. + `restic find` is now able to list the snapshots containing a specific tree or + blob, or even the snapshots that contain blobs belonging to a given pack. A list + of IDs can be given, as long as they all have the same type. - The command `find` can also display the pack IDs the blobs belong to, if the `--show-pack-id` - flag is provided. + The command `find` can also display the pack IDs the blobs belong to, if the + `--show-pack-id` flag is provided. https://github.com/restic/restic/issues/1777 https://github.com/restic/restic/pull/1780 * Enhancement #1876: Display reason why forget keeps snapshots - We've added a column to the list of snapshots `forget` keeps which details the reasons to keep a - particuliar snapshot. This makes debugging policies for forget much easier. Please remember - to always try things out with `--dry-run`! + We've added a column to the list of snapshots `forget` keeps which details the + reasons to keep a particuliar snapshot. This makes debugging policies for forget + much easier. Please remember to always try things out with `--dry-run`! https://github.com/restic/restic/pull/1876 * Enhancement #1891: Accept glob in paths loaded via --files-from - Before that, behaviour was different if paths were appended to command line or from a file, - because wild card characters were expanded by shell if appended to command line, but not - expanded if loaded from file. + Before that, behaviour was different if paths were appended to command line or + from a file, because wild card characters were expanded by shell if appended to + command line, but not expanded if loaded from file. https://github.com/restic/restic/issues/1891 * Enhancement #1909: Reject files/dirs by name first - The current scanner/archiver code had an architectural limitation: it always ran the - `lstat()` system call on all files and directories before a decision to include/exclude the - file/dir was made. This lead to a lot of unnecessary system calls for items that could have been - rejected by their name or path only. + The current scanner/archiver code had an architectural limitation: it always ran + the `lstat()` system call on all files and directories before a decision to + include/exclude the file/dir was made. This lead to a lot of unnecessary system + calls for items that could have been rejected by their name or path only. - We've changed the archiver/scanner implementation so that it now first rejects by name/path, - and only runs the system call on the remaining items. This reduces the number of `lstat()` - system calls a lot (depending on the exclude settings). + We've changed the archiver/scanner implementation so that it now first rejects + by name/path, and only runs the system call on the remaining items. This reduces + the number of `lstat()` system calls a lot (depending on the exclude settings). https://github.com/restic/restic/issues/1909 https://github.com/restic/restic/pull/1912 * Enhancement #1920: Vendor dependencies with Go 1.11 Modules - Until now, we've used `dep` for managing dependencies, we've now switch to using Go modules. - For users this does not change much, only if you want to compile restic without downloading - anything with Go 1.11, then you need to run: `go build -mod=vendor build.go` + Until now, we've used `dep` for managing dependencies, we've now switch to using + Go modules. For users this does not change much, only if you want to compile + restic without downloading anything with Go 1.11, then you need to run: `go + build -mod=vendor build.go` https://github.com/restic/restic/pull/1920 * Enhancement #1940: Add directory filter to ls command - The ls command can now be filtered by directories, so that only files in the given directories - will be shown. If the --recursive flag is specified, then ls will traverse subfolders and list - their files as well. + The ls command can now be filtered by directories, so that only files in the + given directories will be shown. If the --recursive flag is specified, then ls + will traverse subfolders and list their files as well. - It used to be possible to specify multiple snapshots, but that has been replaced by only one - snapshot and the possibility of specifying multiple directories. + It used to be possible to specify multiple snapshots, but that has been replaced + by only one snapshot and the possibility of specifying multiple directories. - Specifying directories constrains the walk, which can significantly speed up the listing. + Specifying directories constrains the walk, which can significantly speed up the + listing. https://github.com/restic/restic/issues/1940 https://github.com/restic/restic/pull/1941 * Enhancement #1949: Add new command `self-update` - We have added a new command called `self-update` which downloads the latest released version - of restic from GitHub and replaces the current binary with it. It does not rely on any external - program (so it'll work everywhere), but still verifies the GPG signature using the embedded - GPG public key. + We have added a new command called `self-update` which downloads the latest + released version of restic from GitHub and replaces the current binary with it. + It does not rely on any external program (so it'll work everywhere), but still + verifies the GPG signature using the embedded GPG public key. - By default, the `self-update` command is hidden behind the `selfupdate` built tag, which is - only set when restic is built using `build.go` (including official releases). The reason for - this is that downstream distributions will then not include the command by default, so users - are encouraged to use the platform-specific distribution mechanism. + By default, the `self-update` command is hidden behind the `selfupdate` built + tag, which is only set when restic is built using `build.go` (including official + releases). The reason for this is that downstream distributions will then not + include the command by default, so users are encouraged to use the + platform-specific distribution mechanism. https://github.com/restic/restic/pull/1949 * Enhancement #1953: Ls: Add JSON output support for restic ls cmd - We've implemented listing files in the repository with JSON as output, just pass `--json` as an - option to `restic ls`. This makes the output of the command machine readable. + We've implemented listing files in the repository with JSON as output, just pass + `--json` as an option to `restic ls`. This makes the output of the command + machine readable. https://github.com/restic/restic/pull/1953 * Enhancement #1962: Stream JSON output for ls command - The `ls` command now supports JSON output with the global `--json` flag, and this change - streams out JSON messages one object at a time rather than en entire array buffered in memory - before encoding. The advantage is it allows large listings to be handled efficiently. + The `ls` command now supports JSON output with the global `--json` flag, and + this change streams out JSON messages one object at a time rather than en entire + array buffered in memory before encoding. The advantage is it allows large + listings to be handled efficiently. - Two message types are printed: snapshots and nodes. A snapshot object will precede node - objects which belong to that snapshot. The `struct_type` field can be used to determine which - kind of message an object is. + Two message types are printed: snapshots and nodes. A snapshot object will + precede node objects which belong to that snapshot. The `struct_type` field can + be used to determine which kind of message an object is. https://github.com/restic/restic/pull/1962 * Enhancement #1967: Use `--host` everywhere - We now use the flag `--host` for all commands which need a host name, using `--hostname` (e.g. - for `restic backup`) still works, but will print a deprecation warning. Also, add the short - option `-H` where possible. + We now use the flag `--host` for all commands which need a host name, using + `--hostname` (e.g. for `restic backup`) still works, but will print a + deprecation warning. Also, add the short option `-H` where possible. https://github.com/restic/restic/issues/1967 * Enhancement #2028: Display size of cache directories - The `cache` command now by default shows the size of the individual cache directories. It can be - disabled with `--no-size`. + The `cache` command now by default shows the size of the individual cache + directories. It can be disabled with `--no-size`. https://github.com/restic/restic/issues/2028 https://github.com/restic/restic/pull/2033 @@ -3916,23 +4170,25 @@ restic users. The changes are ordered by importance. * Bugfix #1854: Allow saving files/dirs on different fs with `--one-file-system` - Restic now allows saving files/dirs on a different file system in a subdir correctly even when - `--one-file-system` is specified. + Restic now allows saving files/dirs on a different file system in a subdir + correctly even when `--one-file-system` is specified. The first thing the restic archiver code does is to build a tree of the target - files/directories. If it detects that a parent directory is already included (e.g. `restic - backup /foo /foo/bar/baz`), it'll ignore the latter argument. + files/directories. If it detects that a parent directory is already included + (e.g. `restic backup /foo /foo/bar/baz`), it'll ignore the latter argument. - Without `--one-file-system`, that's perfectly valid: If `/foo` is to be archived, it will - include `/foo/bar/baz`. But with `--one-file-system`, `/foo/bar/baz` may reside on a - different file system, so it won't be included with `/foo`. + Without `--one-file-system`, that's perfectly valid: If `/foo` is to be + archived, it will include `/foo/bar/baz`. But with `--one-file-system`, + `/foo/bar/baz` may reside on a different file system, so it won't be included + with `/foo`. https://github.com/restic/restic/issues/1854 https://github.com/restic/restic/pull/1855 * Bugfix #1861: Fix case-insensitive search with restic find - We've fixed the behavior for `restic find -i PATTERN`, which was broken in v0.9.1. + We've fixed the behavior for `restic find -i PATTERN`, which was broken in + v0.9.1. https://github.com/restic/restic/pull/1861 @@ -3945,21 +4201,22 @@ restic users. The changes are ordered by importance. * Bugfix #1880: Use `--cache-dir` argument for `check` command - `check` command now uses a temporary sub-directory of the specified directory if set using the - `--cache-dir` argument. If not set, the cache directory is created in the default temporary - directory as before. In either case a temporary cache is used to ensure the actual repository is - checked (rather than a local copy). + `check` command now uses a temporary sub-directory of the specified directory if + set using the `--cache-dir` argument. If not set, the cache directory is created + in the default temporary directory as before. In either case a temporary cache + is used to ensure the actual repository is checked (rather than a local copy). - The `--cache-dir` argument was not used by the `check` command, instead a cache directory was - created in the temporary directory. + The `--cache-dir` argument was not used by the `check` command, instead a cache + directory was created in the temporary directory. https://github.com/restic/restic/issues/1880 * Bugfix #1893: Return error when exclude file cannot be read - A bug was found: when multiple exclude files were passed to restic and one of them could not be - read, an error was printed and restic continued, ignoring even the existing exclude files. - Now, an error message is printed and restic aborts when an exclude file cannot be read. + A bug was found: when multiple exclude files were passed to restic and one of + them could not be read, an error was printed and restic continued, ignoring even + the existing exclude files. Now, an error message is printed and restic aborts + when an exclude file cannot be read. https://github.com/restic/restic/issues/1893 @@ -3970,9 +4227,9 @@ restic users. The changes are ordered by importance. * Enhancement #1477: S3 backend: accept AWS_SESSION_TOKEN - Before, it was not possible to use s3 backend with AWS temporary security credentials(with - AWS_SESSION_TOKEN). This change gives higher priority to credentials.EnvAWS credentials - provider. + Before, it was not possible to use s3 backend with AWS temporary security + credentials(with AWS_SESSION_TOKEN). This change gives higher priority to + credentials.EnvAWS credentials provider. https://github.com/restic/restic/issues/1477 https://github.com/restic/restic/pull/1479 @@ -3980,33 +4237,33 @@ restic users. The changes are ordered by importance. * Enhancement #1772: Add restore --verify to verify restored file content - Restore will print error message if restored file content does not match expected SHA256 - checksum + Restore will print error message if restored file content does not match + expected SHA256 checksum https://github.com/restic/restic/pull/1772 * Enhancement #1853: Add JSON output support to `restic key list` - This PR enables users to get the output of `restic key list` in JSON in addition to the existing - table format. + This PR enables users to get the output of `restic key list` in JSON in addition + to the existing table format. https://github.com/restic/restic/pull/1853 * Enhancement #1901: Update the Backblaze B2 library - We've updated the library we're using for accessing the Backblaze B2 service to 0.5.0 to - include support for upcoming so-called "application keys". With this feature, you can create - access credentials for B2 which are restricted to e.g. a single bucket or even a sub-directory - of a bucket. + We've updated the library we're using for accessing the Backblaze B2 service to + 0.5.0 to include support for upcoming so-called "application keys". With this + feature, you can create access credentials for B2 which are restricted to e.g. a + single bucket or even a sub-directory of a bucket. https://github.com/restic/restic/pull/1901 https://github.com/kurin/blazer * Enhancement #1906: Add support for B2 application keys - Restic can now use so-called "application keys" which can be created in the B2 dashboard and - were only introduced recently. In contrast to the "master key", such keys can be restricted to a - specific bucket and/or path. + Restic can now use so-called "application keys" which can be created in the B2 + dashboard and were only introduced recently. In contrast to the "master key", + such keys can be restricted to a specific bucket and/or path. https://github.com/restic/restic/issues/1906 https://github.com/restic/restic/pull/1914 @@ -4028,48 +4285,51 @@ restic users. The changes are ordered by importance. * Bugfix #1801: Add limiting bandwidth to the rclone backend - The rclone backend did not respect `--limit-upload` or `--limit-download`. Oftentimes it's - not necessary to use this, as the limiting in rclone itself should be used because it gives much - better results, but in case a remote instance of rclone is used (e.g. called via ssh), it is still - relevant to limit the bandwidth from restic to rclone. + The rclone backend did not respect `--limit-upload` or `--limit-download`. + Oftentimes it's not necessary to use this, as the limiting in rclone itself + should be used because it gives much better results, but in case a remote + instance of rclone is used (e.g. called via ssh), it is still relevant to limit + the bandwidth from restic to rclone. https://github.com/restic/restic/issues/1801 * Bugfix #1822: Allow uploading large files to MS Azure - Sometimes, restic creates files to be uploaded to the repository which are quite large, e.g. - when saving directories with many entries or very large files. The MS Azure API does not allow - uploading files larger that 256MiB directly, rather restic needs to upload them in blocks of - 100MiB. This is now implemented. + Sometimes, restic creates files to be uploaded to the repository which are quite + large, e.g. when saving directories with many entries or very large files. The + MS Azure API does not allow uploading files larger that 256MiB directly, rather + restic needs to upload them in blocks of 100MiB. This is now implemented. https://github.com/restic/restic/issues/1822 * Bugfix #1825: Correct `find` to not skip snapshots - Under certain circumstances, the `find` command was found to skip snapshots containing - directories with files to look for when the directories haven't been modified at all, and were - already printed as part of a different snapshot. This is now corrected. + Under certain circumstances, the `find` command was found to skip snapshots + containing directories with files to look for when the directories haven't been + modified at all, and were already printed as part of a different snapshot. This + is now corrected. - In addition, we've switched to our own matching/pattern implementation, so now things like - `restic find "/home/user/foo/**/main.go"` are possible. + In addition, we've switched to our own matching/pattern implementation, so now + things like `restic find "/home/user/foo/**/main.go"` are possible. https://github.com/restic/restic/issues/1825 https://github.com/restic/restic/issues/1823 * Bugfix #1833: Fix caching files on error - During `check` it may happen that different threads access the same file in the backend, which - is then downloaded into the cache only once. When that fails, only the thread which is - responsible for downloading the file signals the correct error. The other threads just assume - that the file has been downloaded successfully and then get an error when they try to access the - cached file. + During `check` it may happen that different threads access the same file in the + backend, which is then downloaded into the cache only once. When that fails, + only the thread which is responsible for downloading the file signals the + correct error. The other threads just assume that the file has been downloaded + successfully and then get an error when they try to access the cached file. https://github.com/restic/restic/issues/1833 * Bugfix #1834: Resolve deadlock - When the "scanning" process restic runs to find out how much data there is does not finish before - the backup itself is done, restic stops doing anything. This is resolved now. + When the "scanning" process restic runs to find out how much data there is does + not finish before the backup itself is done, restic stops doing anything. This + is resolved now. https://github.com/restic/restic/issues/1834 https://github.com/restic/restic/pull/1835 @@ -4105,77 +4365,81 @@ restic users. The changes are ordered by importance. * Bugfix #1608: Respect time stamp for new backup when reading from stdin - When reading backups from stdin (via `restic backup --stdin`), restic now uses the time stamp - for the new backup passed in `--time`. + When reading backups from stdin (via `restic backup --stdin`), restic now uses + the time stamp for the new backup passed in `--time`. https://github.com/restic/restic/issues/1608 https://github.com/restic/restic/pull/1703 * Bugfix #1652: Ignore/remove invalid lock files - This corrects a bug introduced recently: When an invalid lock file in the repo is encountered - (e.g. if the file is empty), the code used to ignore that, but now returns the error. Now, invalid - files are ignored for the normal lock check, and removed when `restic unlock --remove-all` is - run. + This corrects a bug introduced recently: When an invalid lock file in the repo + is encountered (e.g. if the file is empty), the code used to ignore that, but + now returns the error. Now, invalid files are ignored for the normal lock check, + and removed when `restic unlock --remove-all` is run. https://github.com/restic/restic/issues/1652 https://github.com/restic/restic/pull/1653 * Bugfix #1684: Fix backend tests for rest-server - The REST server for restic now requires an explicit parameter (`--no-auth`) if no - authentication should be allowed. This is fixed in the tests. + The REST server for restic now requires an explicit parameter (`--no-auth`) if + no authentication should be allowed. This is fixed in the tests. https://github.com/restic/restic/pull/1684 * Bugfix #1730: Ignore sockets for restore - We've received a report and correct the behavior in which the restore code aborted restoring a - directory when a socket was encountered. Unix domain socket files cannot be restored (they are - created on the fly once a process starts listening). The error handling was corrected, and in - addition we're now ignoring sockets during restore. + We've received a report and correct the behavior in which the restore code + aborted restoring a directory when a socket was encountered. Unix domain socket + files cannot be restored (they are created on the fly once a process starts + listening). The error handling was corrected, and in addition we're now ignoring + sockets during restore. https://github.com/restic/restic/issues/1730 https://github.com/restic/restic/pull/1731 * Bugfix #1745: Correctly parse the argument to --tls-client-cert - Previously, the --tls-client-cert method attempt to read ARGV[1] (hardcoded) instead of the - argument that was passed to it. This has been corrected. + Previously, the --tls-client-cert method attempt to read ARGV[1] (hardcoded) + instead of the argument that was passed to it. This has been corrected. https://github.com/restic/restic/issues/1745 https://github.com/restic/restic/pull/1746 * Enhancement #549: Rework archiver code - The core archiver code and the complementary code for the `backup` command was rewritten - completely. This resolves very annoying issues such as 549. The first backup with this release - of restic will likely result in all files being re-read locally, so it will take a lot longer. The - next backup after that will be fast again. + The core archiver code and the complementary code for the `backup` command was + rewritten completely. This resolves very annoying issues such as 549. The first + backup with this release of restic will likely result in all files being re-read + locally, so it will take a lot longer. The next backup after that will be fast + again. - Basically, with the old code, restic took the last path component of each to-be-saved file or - directory as the top-level file/directory within the snapshot. This meant that when called as - `restic backup /home/user/foo`, the snapshot would contain the files in the directory - `/home/user/foo` as `/foo`. + Basically, with the old code, restic took the last path component of each + to-be-saved file or directory as the top-level file/directory within the + snapshot. This meant that when called as `restic backup /home/user/foo`, the + snapshot would contain the files in the directory `/home/user/foo` as `/foo`. - This is not the case any more with the new archiver code. Now, restic works very similar to what - `tar` does: When restic is called with an absolute path to save, then it'll preserve the - directory structure within the snapshot. For the example above, the snapshot would contain - the files in the directory within `/home/user/foo` in the snapshot. For relative - directories, it only preserves the relative path components. So `restic backup user/foo` - will save the files as `/user/foo` in the snapshot. + This is not the case any more with the new archiver code. Now, restic works very + similar to what `tar` does: When restic is called with an absolute path to save, + then it'll preserve the directory structure within the snapshot. For the example + above, the snapshot would contain the files in the directory within + `/home/user/foo` in the snapshot. For relative directories, it only preserves + the relative path components. So `restic backup user/foo` will save the files as + `/user/foo` in the snapshot. - While we were at it, the status display and notification system was completely rewritten. By - default, restic now shows which files are currently read (unless `--quiet` is specified) in a - multi-line status display. + While we were at it, the status display and notification system was completely + rewritten. By default, restic now shows which files are currently read (unless + `--quiet` is specified) in a multi-line status display. - The `backup` command also gained a new option: `--verbose`. It can be specified once (which - prints a bit more detail what restic is doing) or twice (which prints a line for each - file/directory restic encountered, together with some statistics). + The `backup` command also gained a new option: `--verbose`. It can be specified + once (which prints a bit more detail what restic is doing) or twice (which + prints a line for each file/directory restic encountered, together with some + statistics). - Another issue that was resolved is the new code only reads two files at most. The old code would - read way too many files in parallel, thereby slowing down the backup process on spinning discs a - lot. + Another issue that was resolved is the new code only reads two files at most. + The old code would read way too many files in parallel, thereby slowing down the + backup process on spinning discs a lot. https://github.com/restic/restic/issues/549 https://github.com/restic/restic/issues/1286 @@ -4197,11 +4461,11 @@ restic users. The changes are ordered by importance. * Enhancement #1433: Support UTF-16 encoding and process Byte Order Mark - On Windows, text editors commonly leave a Byte Order Mark at the beginning of the file to define - which encoding is used (oftentimes UTF-16). We've added code to support processing the BOMs in - text files, like the exclude files, the password file and the file passed via `--files-from`. - This does not apply to any file being saved in a backup, those are not touched and archived as they - are. + On Windows, text editors commonly leave a Byte Order Mark at the beginning of + the file to define which encoding is used (oftentimes UTF-16). We've added code + to support processing the BOMs in text files, like the exclude files, the + password file and the file passed via `--files-from`. This does not apply to any + file being saved in a backup, those are not touched and archived as they are. https://github.com/restic/restic/issues/1433 https://github.com/restic/restic/issues/1738 @@ -4209,9 +4473,9 @@ restic users. The changes are ordered by importance. * Enhancement #1477: Accept AWS_SESSION_TOKEN for the s3 backend - Before, it was not possible to use s3 backend with AWS temporary security credentials(with - AWS_SESSION_TOKEN). This change gives higher priority to credentials.EnvAWS credentials - provider. + Before, it was not possible to use s3 backend with AWS temporary security + credentials(with AWS_SESSION_TOKEN). This change gives higher priority to + credentials.EnvAWS credentials provider. https://github.com/restic/restic/issues/1477 https://github.com/restic/restic/pull/1479 @@ -4219,23 +4483,24 @@ restic users. The changes are ordered by importance. * Enhancement #1552: Use Google Application Default credentials - Google provide libraries to generate appropriate credentials with various fallback - sources. This change uses the library to generate our GCS client, which allows us to make use of - these extra methods. + Google provide libraries to generate appropriate credentials with various + fallback sources. This change uses the library to generate our GCS client, which + allows us to make use of these extra methods. - This should be backward compatible with previous restic behaviour while adding the - additional capabilities to auth from Google's internal metadata endpoints. For users - running restic in GCP this can make authentication far easier than it was before. + This should be backward compatible with previous restic behaviour while adding + the additional capabilities to auth from Google's internal metadata endpoints. + For users running restic in GCP this can make authentication far easier than it + was before. https://github.com/restic/restic/pull/1552 https://developers.google.com/identity/protocols/application-default-credentials * Enhancement #1561: Allow using rclone to access other services - We've added the ability to use rclone to store backup data on all backends that it supports. This - was done in collaboration with Nick, the author of rclone. You can now use it to first configure a - service, then restic manages the rest (starting and stopping rclone). For details, please see - the manual. + We've added the ability to use rclone to store backup data on all backends that + it supports. This was done in collaboration with Nick, the author of rclone. You + can now use it to first configure a service, then restic manages the rest + (starting and stopping rclone). For details, please see the manual. https://github.com/restic/restic/issues/1561 https://github.com/restic/restic/pull/1657 @@ -4243,9 +4508,9 @@ restic users. The changes are ordered by importance. * Enhancement #1648: Ignore AWS permission denied error when creating a repository - It's not possible to use s3 backend scoped to a subdirectory(with specific permissions). - Restic doesn't try to create repository in a subdirectory, when 'bucket exists' of parent - directory check fails due to permission issues. + It's not possible to use s3 backend scoped to a subdirectory(with specific + permissions). Restic doesn't try to create repository in a subdirectory, when + 'bucket exists' of parent directory check fails due to permission issues. https://github.com/restic/restic/pull/1648 @@ -4255,25 +4520,27 @@ restic users. The changes are ordered by importance. * Enhancement #1665: Improve cache handling for `restic check` - For safety reasons, restic does not use a local metadata cache for the `restic check` command, - so that data is loaded from the repository and restic can check it's in good condition. When the - cache is disabled, restic will fetch each tiny blob needed for checking the integrity using a - separate backend request. For non-local backends, that will take a long time, and depending on - the backend (e.g. B2) may also be much more expensive. + For safety reasons, restic does not use a local metadata cache for the `restic + check` command, so that data is loaded from the repository and restic can check + it's in good condition. When the cache is disabled, restic will fetch each tiny + blob needed for checking the integrity using a separate backend request. For + non-local backends, that will take a long time, and depending on the backend + (e.g. B2) may also be much more expensive. This PR adds a few commits which will change the behavior as follows: - * When `restic check` is called without any additional parameters, it will build a new cache in a - temporary directory, which is removed at the end of the check. This way, we'll get readahead for - metadata files (so restic will fetch the whole file when the first blob from the file is - requested), but all data is freshly fetched from the storage backend. This is the default - behavior and will work for almost all users. + * When `restic check` is called without any additional parameters, it will build + a new cache in a temporary directory, which is removed at the end of the check. + This way, we'll get readahead for metadata files (so restic will fetch the whole + file when the first blob from the file is requested), but all data is freshly + fetched from the storage backend. This is the default behavior and will work for + almost all users. - * When `restic check` is called with `--with-cache`, the default on-disc cache is used. This - behavior hasn't changed since the cache was introduced. + * When `restic check` is called with `--with-cache`, the default on-disc cache + is used. This behavior hasn't changed since the cache was introduced. - * When `--no-cache` is specified, restic falls back to the old behavior, and read all tiny blobs - in separate requests. + * When `--no-cache` is specified, restic falls back to the old behavior, and + read all tiny blobs in separate requests. https://github.com/restic/restic/issues/1665 https://github.com/restic/restic/issues/1694 @@ -4281,44 +4548,45 @@ restic users. The changes are ordered by importance. * Enhancement #1709: Improve messages `restic check` prints - Some messages `restic check` prints are not really errors, so from now on restic does not treat - them as errors any more and exits cleanly. + Some messages `restic check` prints are not really errors, so from now on restic + does not treat them as errors any more and exits cleanly. https://github.com/restic/restic/pull/1709 https://forum.restic.net/t/what-is-the-standard-procedure-to-follow-if-a-backup-or-restore-is-interrupted/571/2 * Enhancement #1721: Add `cache` command to list cache dirs - The command `cache` was added, it allows listing restic's cache directoriers together with - the last usage. It also allows removing old cache dirs without having to access a repo, via - `restic cache --cleanup` + The command `cache` was added, it allows listing restic's cache directoriers + together with the last usage. It also allows removing old cache dirs without + having to access a repo, via `restic cache --cleanup` https://github.com/restic/restic/issues/1721 https://github.com/restic/restic/pull/1749 * Enhancement #1735: Allow keeping a time range of snaphots - We've added the `--keep-within` option to the `forget` command. It instructs restic to keep - all snapshots within the given duration since the newest snapshot. For example, running - `restic forget --keep-within 5m7d` will keep all snapshots which have been made in the five - months and seven days since the latest snapshot. + We've added the `--keep-within` option to the `forget` command. It instructs + restic to keep all snapshots within the given duration since the newest + snapshot. For example, running `restic forget --keep-within 5m7d` will keep all + snapshots which have been made in the five months and seven days since the + latest snapshot. https://github.com/restic/restic/pull/1735 * Enhancement #1758: Allow saving OneDrive folders in Windows - Restic now contains a bugfix to two libraries, which allows saving OneDrive folders in - Windows. In order to use the newer versions of the libraries, the minimal version required to - compile restic is now Go 1.9. + Restic now contains a bugfix to two libraries, which allows saving OneDrive + folders in Windows. In order to use the newer versions of the libraries, the + minimal version required to compile restic is now Go 1.9. https://github.com/restic/restic/issues/1758 https://github.com/restic/restic/pull/1765 * Enhancement #1782: Use default AWS credentials chain for S3 backend - Adds support for file credentials to the S3 backend (e.g. ~/.aws/credentials), and reorders - the credentials chain for the S3 backend to match AWS's standard, which is static credentials, - env vars, credentials file, and finally remote. + Adds support for file credentials to the S3 backend (e.g. ~/.aws/credentials), + and reorders the credentials chain for the S3 backend to match AWS's standard, + which is static credentials, env vars, credentials file, and finally remote. https://github.com/restic/restic/pull/1782 @@ -4341,32 +4609,34 @@ restic users. The changes are ordered by importance. * Bugfix #1633: Fixed unexpected 'pack file cannot be listed' error - Due to a regression introduced in 0.8.2, the `rebuild-index` and `prune` commands failed to - read pack files with size of 587, 588, 589 or 590 bytes. + Due to a regression introduced in 0.8.2, the `rebuild-index` and `prune` + commands failed to read pack files with size of 587, 588, 589 or 590 bytes. https://github.com/restic/restic/issues/1633 https://github.com/restic/restic/pull/1635 * Bugfix #1638: Handle errors listing files in the backend - A user reported in the forum that restic completes a backup although a concurrent `prune` - operation was running. A few error messages were printed, but the backup was attempted and - completed successfully. No error code was returned. + A user reported in the forum that restic completes a backup although a + concurrent `prune` operation was running. A few error messages were printed, but + the backup was attempted and completed successfully. No error code was returned. - This should not happen: The repository is exclusively locked during `prune`, so when `restic - backup` is run in parallel, it should abort and return an error code instead. + This should not happen: The repository is exclusively locked during `prune`, so + when `restic backup` is run in parallel, it should abort and return an error + code instead. - It was found that the bug was in the code introduced only recently, which retries a List() - operation on the backend should that fail. It is now corrected. + It was found that the bug was in the code introduced only recently, which + retries a List() operation on the backend should that fail. It is now corrected. https://github.com/restic/restic/pull/1638 https://forum.restic.net/t/restic-backup-returns-0-exit-code-when-already-locked/484 * Bugfix #1641: Ignore files with invalid names in the repo - The release 0.8.2 introduced a bug: when restic encounters files in the repo which do not have a - valid name, it tries to load a file with a name of lots of zeroes instead of ignoring it. This is now - resolved, invalid file names are just ignored. + The release 0.8.2 introduced a bug: when restic encounters files in the repo + which do not have a valid name, it tries to load a file with a name of lots of + zeroes instead of ignoring it. This is now resolved, invalid file names are just + ignored. https://github.com/restic/restic/issues/1641 https://github.com/restic/restic/pull/1643 @@ -4374,8 +4644,9 @@ restic users. The changes are ordered by importance. * Enhancement #1497: Add --read-data-subset flag to check command - This change introduces ability to check integrity of a subset of repository data packs. This - can be used to spread integrity check of larger repositories over a period of time. + This change introduces ability to check integrity of a subset of repository data + packs. This can be used to spread integrity check of larger repositories over a + period of time. https://github.com/restic/restic/issues/1497 https://github.com/restic/restic/pull/1556 @@ -4388,21 +4659,22 @@ restic users. The changes are ordered by importance. * Enhancement #1623: Don't check for presence of files in the backend before writing - Before, all backend implementations were required to return an error if the file that is to be - written already exists in the backend. For most backends, that means making a request (e.g. via - HTTP) and returning an error when the file already exists. + Before, all backend implementations were required to return an error if the file + that is to be written already exists in the backend. For most backends, that + means making a request (e.g. via HTTP) and returning an error when the file + already exists. - This is not accurate, the file could have been created between the HTTP request testing for it, - and when writing starts, so we've relaxed this requeriment, which saves one additional HTTP - request per newly added file. + This is not accurate, the file could have been created between the HTTP request + testing for it, and when writing starts, so we've relaxed this requeriment, + which saves one additional HTTP request per newly added file. https://github.com/restic/restic/pull/1623 * Enhancement #1634: Upgrade B2 client library, reduce HTTP requests - We've upgraded the B2 client library restic uses to access BackBlaze B2. This reduces the - number of HTTP requests needed to upload a new file from two to one, which should improve - throughput to B2. + We've upgraded the B2 client library restic uses to access BackBlaze B2. This + reduces the number of HTTP requests needed to upload a new file from two to one, + which should improve throughput to B2. https://github.com/restic/restic/pull/1634 @@ -4440,10 +4712,10 @@ restic users. The changes are ordered by importance. * Bugfix #1512: Restore directory permissions as the last step - This change allows restoring into directories that were not writable during backup. Before, - restic created the directory, set the read-only mode and then failed to create files in the - directory. This change now restores the directory (with its permissions) as the very last - step. + This change allows restoring into directories that were not writable during + backup. Before, restic created the directory, set the read-only mode and then + failed to create files in the directory. This change now restores the directory + (with its permissions) as the very last step. https://github.com/restic/restic/issues/1512 https://github.com/restic/restic/pull/1536 @@ -4455,43 +4727,47 @@ restic users. The changes are ordered by importance. * Bugfix #1589: Complete intermediate index upload - After a user posted a comprehensive report of what he observed, we were able to find a bug and - correct it: During backup, restic uploads so-called "intermediate" index files. When the - backup finishes during a transfer of such an intermediate index, the upload is cancelled, but - the backup is finished without an error. This leads to an inconsistent state, where the - snapshot references data that is contained in the repo, but is not referenced in any index. + After a user posted a comprehensive report of what he observed, we were able to + find a bug and correct it: During backup, restic uploads so-called + "intermediate" index files. When the backup finishes during a transfer of such + an intermediate index, the upload is cancelled, but the backup is finished + without an error. This leads to an inconsistent state, where the snapshot + references data that is contained in the repo, but is not referenced in any + index. - The situation can be resolved by building a new index with `rebuild-index`, but looks very - confusing at first. Since all the data got uploaded to the repo successfully, there was no risk - of data loss, just minor inconvenience for our users. + The situation can be resolved by building a new index with `rebuild-index`, but + looks very confusing at first. Since all the data got uploaded to the repo + successfully, there was no risk of data loss, just minor inconvenience for our + users. https://github.com/restic/restic/pull/1589 https://forum.restic.net/t/error-loading-tree-check-prune-and-forget-gives-error-b2-backend/406 * Bugfix #1590: Strip spaces for lines read via --files-from - Leading and trailing spaces in lines read via `--files-from` are now stripped, so it behaves - the same as with lines read via `--exclude-file`. + Leading and trailing spaces in lines read via `--files-from` are now stripped, + so it behaves the same as with lines read via `--exclude-file`. https://github.com/restic/restic/issues/1590 https://github.com/restic/restic/pull/1613 * Bugfix #1594: Google Cloud Storage: Use generic HTTP transport - It was discovered that the Google Cloud Storage backend did not use the generic HTTP transport, - so things such as bandwidth limiting with `--limit-upload` did not work. This is resolved now. + It was discovered that the Google Cloud Storage backend did not use the generic + HTTP transport, so things such as bandwidth limiting with `--limit-upload` did + not work. This is resolved now. https://github.com/restic/restic/pull/1594 * Bugfix #1595: Backup: Remove bandwidth display - This commit removes the bandwidth displayed during backup process. It is misleading and - seldomly correct, because it's neither the "read bandwidth" (only for the very first backup) - nor the "upload bandwidth". Many users are confused about (and rightly so), c.f. #1581, #1033, - #1591 + This commit removes the bandwidth displayed during backup process. It is + misleading and seldomly correct, because it's neither the "read bandwidth" (only + for the very first backup) nor the "upload bandwidth". Many users are confused + about (and rightly so), c.f. #1581, #1033, #1591 - We'll eventually replace this display with something more relevant when the new archiver code - is ready. + We'll eventually replace this display with something more relevant when the new + archiver code is ready. https://github.com/restic/restic/pull/1595 @@ -4501,59 +4777,61 @@ restic users. The changes are ordered by importance. * Enhancement #1522: Add support for TLS client certificate authentication - Support has been added for using a TLS client certificate for authentication to HTTP based - backend. A file containing the PEM encoded private key and certificate can be set using the - `--tls-client-cert` option. + Support has been added for using a TLS client certificate for authentication to + HTTP based backend. A file containing the PEM encoded private key and + certificate can be set using the `--tls-client-cert` option. https://github.com/restic/restic/issues/1522 https://github.com/restic/restic/pull/1524 * Enhancement #1538: Reduce memory allocations for querying the index - This change reduces the internal memory allocations when the index data structures in memory - are queried if a blob (part of a file) already exists in the repo. It should speed up backup a bit, - and maybe even reduce RAM usage. + This change reduces the internal memory allocations when the index data + structures in memory are queried if a blob (part of a file) already exists in + the repo. It should speed up backup a bit, and maybe even reduce RAM usage. https://github.com/restic/restic/pull/1538 * Enhancement #1541: Reduce number of remote requests during repository check - This change eliminates redundant remote repository calls and significantly improves - repository check time. + This change eliminates redundant remote repository calls and significantly + improves repository check time. https://github.com/restic/restic/issues/1541 https://github.com/restic/restic/pull/1548 * Enhancement #1549: Speed up querying across indices and scanning existing files - This change increases the whenever a blob (part of a file) is searched for in a restic - repository. This will reduce cpu usage some when backing up files already backed up by restic. - Cpu usage is further decreased when scanning files. + This change increases the whenever a blob (part of a file) is searched for in a + restic repository. This will reduce cpu usage some when backing up files already + backed up by restic. Cpu usage is further decreased when scanning files. https://github.com/restic/restic/pull/1549 * Enhancement #1554: Fuse/mount: Correctly handle EOF, add template option - We've added the `--snapshot-template` string, which can be used to specify a template for a - snapshot directory. In addition, accessing data after the end of a file via the fuse mount is now - handled correctly. + We've added the `--snapshot-template` string, which can be used to specify a + template for a snapshot directory. In addition, accessing data after the end of + a file via the fuse mount is now handled correctly. https://github.com/restic/restic/pull/1554 * Enhancement #1564: Don't terminate ssh on SIGINT - We've reworked the code which runs the `ssh` login for the sftp backend so that it can prompt for a - password (if needed) but does not exit when the user presses CTRL+C (SIGINT) e.g. during - backup. This allows restic to properly shut down when it receives SIGINT and remove the lock - file from the repo, afterwards exiting the `ssh` process. + We've reworked the code which runs the `ssh` login for the sftp backend so that + it can prompt for a password (if needed) but does not exit when the user presses + CTRL+C (SIGINT) e.g. during backup. This allows restic to properly shut down + when it receives SIGINT and remove the lock file from the repo, afterwards + exiting the `ssh` process. https://github.com/restic/restic/pull/1564 https://github.com/restic/restic/pull/1588 * Enhancement #1567: Reduce number of backend requests for rebuild-index and prune - We've found a way to reduce then number of backend requests for the `rebuild-index` and `prune` - operations. This significantly speeds up the operations for high-latency backends. + We've found a way to reduce then number of backend requests for the + `rebuild-index` and `prune` operations. This significantly speeds up the + operations for high-latency backends. https://github.com/restic/restic/issues/1567 https://github.com/restic/restic/pull/1574 @@ -4565,10 +4843,11 @@ restic users. The changes are ordered by importance. * Enhancement #1584: Limit index file size - Before, restic would create a single new index file on `prune` or `rebuild-index`, this may - lead to memory problems when this huge index is created and loaded again. We're now limiting the - size of the index file, and split newly created index files into several smaller ones. This - allows restic to be more memory-efficient. + Before, restic would create a single new index file on `prune` or + `rebuild-index`, this may lead to memory problems when this huge index is + created and loaded again. We're now limiting the size of the index file, and + split newly created index files into several smaller ones. This allows restic to + be more memory-efficient. https://github.com/restic/restic/issues/1412 https://github.com/restic/restic/issues/979 @@ -4594,8 +4873,8 @@ restic users. The changes are ordered by importance. * Bugfix #1454: Correct cache dir location for Windows and Darwin - The cache directory on Windows and Darwin was not correct, instead the directory `.cache` was - used. + The cache directory on Windows and Darwin was not correct, instead the directory + `.cache` was used. https://github.com/restic/restic/pull/1454 @@ -4606,9 +4885,9 @@ restic users. The changes are ordered by importance. * Bugfix #1459: Disable handling SIGPIPE - We've disabled handling SIGPIPE again. Turns out, writing to broken TCP connections also - raised SIGPIPE, so restic exits on the first write to a broken connection. Instead, restic - should retry the request. + We've disabled handling SIGPIPE again. Turns out, writing to broken TCP + connections also raised SIGPIPE, so restic exits on the first write to a broken + connection. Instead, restic should retry the request. https://github.com/restic/restic/issues/1457 https://github.com/restic/restic/issues/1466 @@ -4616,16 +4895,18 @@ restic users. The changes are ordered by importance. * Change #1452: Do not save atime by default - By default, the access time for files and dirs is not saved any more. It is not possible to - reliably disable updating the access time during a backup, so for the next backup the access - time is different again. This means a lot of metadata is saved. If you want to save the access time - anyway, pass `--with-atime` to the `backup` command. + By default, the access time for files and dirs is not saved any more. It is not + possible to reliably disable updating the access time during a backup, so for + the next backup the access time is different again. This means a lot of metadata + is saved. If you want to save the access time anyway, pass `--with-atime` to the + `backup` command. https://github.com/restic/restic/pull/1452 * Enhancement #11: Add the `diff` command - The command `diff` was added, it allows comparing two snapshots and listing all differences. + The command `diff` was added, it allows comparing two snapshots and listing all + differences. https://github.com/restic/restic/issues/11 https://github.com/restic/restic/issues/1460 @@ -4633,17 +4914,18 @@ restic users. The changes are ordered by importance. * Enhancement #1436: Add code to detect old cache directories - We've added code to detect old cache directories of repositories that haven't been used in a - long time, restic now prints a note when it detects that such dirs exist. Also, the option - `--cleanup-cache` was added to automatically remove such directories. That's not a problem - because the cache will be rebuild once a repo is accessed again. + We've added code to detect old cache directories of repositories that haven't + been used in a long time, restic now prints a note when it detects that such + dirs exist. Also, the option `--cleanup-cache` was added to automatically remove + such directories. That's not a problem because the cache will be rebuild once a + repo is accessed again. https://github.com/restic/restic/pull/1436 * Enhancement #1439: Improve cancellation logic - The cancellation logic was improved, restic can now shut down cleanly when requested to do so - (e.g. via ctrl+c). + The cancellation logic was improved, restic can now shut down cleanly when + requested to do so (e.g. via ctrl+c). https://github.com/restic/restic/pull/1439 @@ -4678,17 +4960,18 @@ restic users. The changes are ordered by importance. * Security #1445: Prevent writing outside the target directory during restore - A vulnerability was found in the restic restorer, which allowed attackers in special - circumstances to restore files to a location outside of the target directory. Due to the - circumstances we estimate this to be a low-risk vulnerability, but urge all users to upgrade to - the latest version of restic. + A vulnerability was found in the restic restorer, which allowed attackers in + special circumstances to restore files to a location outside of the target + directory. Due to the circumstances we estimate this to be a low-risk + vulnerability, but urge all users to upgrade to the latest version of restic. - Exploiting the vulnerability requires a Linux/Unix system which saves backups via restic and - a Windows systems which restores files from the repo. In addition, the attackers need to be able - to create files with arbitrary names which are then saved to the restic repo. For example, by - creating a file named "..\test.txt" (which is a perfectly legal filename on Linux) and - restoring a snapshot containing this file on Windows, it would be written to the parent of the - target directory. + Exploiting the vulnerability requires a Linux/Unix system which saves backups + via restic and a Windows systems which restores files from the repo. In + addition, the attackers need to be able to create files with arbitrary names + which are then saved to the restic repo. For example, by creating a file named + "..\test.txt" (which is a perfectly legal filename on Linux) and restoring a + snapshot containing this file on Windows, it would be written to the parent of + the target directory. We'd like to thank Tyler Spivey for reporting this responsibly! @@ -4696,34 +4979,36 @@ restic users. The changes are ordered by importance. * Bugfix #1256: Re-enable workaround for S3 backend - We've re-enabled a workaround for `minio-go` (the library we're using to access s3 backends), - this reduces memory usage. + We've re-enabled a workaround for `minio-go` (the library we're using to access + s3 backends), this reduces memory usage. https://github.com/restic/restic/issues/1256 https://github.com/restic/restic/pull/1267 * Bugfix #1291: Reuse backend TCP connections to BackBlaze B2 - A bug was discovered in the library we're using to access Backblaze, it now reuses already - established TCP connections which should be a lot faster and not cause network failures any - more. + A bug was discovered in the library we're using to access Backblaze, it now + reuses already established TCP connections which should be a lot faster and not + cause network failures any more. https://github.com/restic/restic/issues/1291 https://github.com/restic/restic/pull/1301 * Bugfix #1317: Run prune when `forget --prune` is called with just snapshot IDs - A bug in the `forget` command caused `prune` not to be run when `--prune` was specified without a - policy, e.g. when only snapshot IDs that should be forgotten are listed manually. + A bug in the `forget` command caused `prune` not to be run when `--prune` was + specified without a policy, e.g. when only snapshot IDs that should be forgotten + are listed manually. https://github.com/restic/restic/pull/1317 * Bugfix #1437: Remove implicit path `/restic` for the s3 backend - The s3 backend used the subdir `restic` within a bucket if no explicit path after the bucket name - was specified. Since this version, restic does not use this default path any more. If you - created a repo on s3 in a bucket without specifying a path within the bucket, you need to add - `/restic` at the end of the repository specification to access your repo: + The s3 backend used the subdir `restic` within a bucket if no explicit path + after the bucket name was specified. Since this version, restic does not use + this default path any more. If you created a repo on s3 in a bucket without + specifying a path within the bucket, you need to add `/restic` at the end of the + repository specification to access your repo: `s3:s3.amazonaws.com/bucket/restic` https://github.com/restic/restic/issues/1292 @@ -4731,32 +5016,35 @@ restic users. The changes are ordered by importance. * Enhancement #448: Sftp backend prompts for password - The sftp backend now prompts for the password if a password is necessary for login. + The sftp backend now prompts for the password if a password is necessary for + login. https://github.com/restic/restic/issues/448 https://github.com/restic/restic/pull/1270 * Enhancement #510: Add `dump` command - We've added the `dump` command which prints a file from a snapshot to stdout. This can e.g. be - used to restore files read with `backup --stdin`. + We've added the `dump` command which prints a file from a snapshot to stdout. + This can e.g. be used to restore files read with `backup --stdin`. https://github.com/restic/restic/issues/510 https://github.com/restic/restic/pull/1346 * Enhancement #1040: Add local metadata cache - We've added a local cache for metadata so that restic doesn't need to load all metadata - (snapshots, indexes, ...) from the repo each time it starts. By default the cache is active, but - there's a new global option `--no-cache` that can be used to disable the cache. By deafult, the - cache a standard cache folder for the OS, which can be overridden with `--cache-dir`. The cache - will automatically populate, indexes and snapshots are saved as they are loaded. Cache - directories for repos that haven't been used recently can automatically be removed by restic + We've added a local cache for metadata so that restic doesn't need to load all + metadata (snapshots, indexes, ...) from the repo each time it starts. By default + the cache is active, but there's a new global option `--no-cache` that can be + used to disable the cache. By deafult, the cache a standard cache folder for the + OS, which can be overridden with `--cache-dir`. The cache will automatically + populate, indexes and snapshots are saved as they are loaded. Cache directories + for repos that haven't been used recently can automatically be removed by restic with the `--cleanup-cache` option. - A related change was to by default create pack files in the repo that contain either data or - metadata, not both mixed together. This allows easy caching of only the metadata files. The - next run of `restic prune` will untangle mixed files automatically. + A related change was to by default create pack files in the repo that contain + either data or metadata, not both mixed together. This allows easy caching of + only the metadata files. The next run of `restic prune` will untangle mixed + files automatically. https://github.com/restic/restic/issues/29 https://github.com/restic/restic/issues/738 @@ -4768,8 +5056,8 @@ restic users. The changes are ordered by importance. * Enhancement #1102: Add subdirectory `ids` to fuse mount - The fuse mount now has an `ids` subdirectory which contains the snapshots below their (short) - IDs. + The fuse mount now has an `ids` subdirectory which contains the snapshots below + their (short) IDs. https://github.com/restic/restic/issues/1102 https://github.com/restic/restic/pull/1299 @@ -4777,17 +5065,17 @@ restic users. The changes are ordered by importance. * Enhancement #1114: Add `--cacert` to specify TLS certificates to check against - We've added the `--cacert` option which can be used to pass one (or more) CA certificates to - restic. These are used in addition to the system CA certificates to verify HTTPS certificates - (e.g. for the REST backend). + We've added the `--cacert` option which can be used to pass one (or more) CA + certificates to restic. These are used in addition to the system CA certificates + to verify HTTPS certificates (e.g. for the REST backend). https://github.com/restic/restic/issues/1114 https://github.com/restic/restic/pull/1276 * Enhancement #1216: Add upload/download limiting - We've added support for rate limiting through `--limit-upload` and `--limit-download` - flags. + We've added support for rate limiting through `--limit-upload` and + `--limit-download` flags. https://github.com/restic/restic/issues/1216 https://github.com/restic/restic/pull/1336 @@ -4795,15 +5083,15 @@ restic users. The changes are ordered by importance. * Enhancement #1249: Add `latest` symlink in fuse mount - The directory structure in the fuse mount now exposes a symlink `latest` which points to the - latest snapshot in that particular directory. + The directory structure in the fuse mount now exposes a symlink `latest` which + points to the latest snapshot in that particular directory. https://github.com/restic/restic/pull/1249 * Enhancement #1269: Add `--compact` to `forget` command - The option `--compact` was added to the `forget` command to provide the same compact view as the - `snapshots` command. + The option `--compact` was added to the `forget` command to provide the same + compact view as the `snapshots` command. https://github.com/restic/restic/pull/1269 @@ -4816,25 +5104,26 @@ restic users. The changes are ordered by importance. * Enhancement #1274: Add `generate` command, replaces `manpage` and `autocomplete` - The `generate` command has been added, which replaces the now removed commands `manpage` and - `autocomplete`. This release of restic contains the most recent manpages in `doc/man` and the - auto-completion files for bash and zsh in `doc/bash-completion.sh` and - `doc/zsh-completion.zsh` + The `generate` command has been added, which replaces the now removed commands + `manpage` and `autocomplete`. This release of restic contains the most recent + manpages in `doc/man` and the auto-completion files for bash and zsh in + `doc/bash-completion.sh` and `doc/zsh-completion.zsh` https://github.com/restic/restic/issues/1274 https://github.com/restic/restic/pull/1282 * Enhancement #1281: Google Cloud Storage backend needs less permissions - The Google Cloud Storage backend no longer requires the service account to have the - `storage.buckets.get` permission ("Storage Admin" role) in `restic init` if the bucket - already exists. + The Google Cloud Storage backend no longer requires the service account to have + the `storage.buckets.get` permission ("Storage Admin" role) in `restic init` if + the bucket already exists. https://github.com/restic/restic/pull/1281 * Enhancement #1319: Make `check` print `no errors found` explicitly - The `check` command now explicetly prints `No errors were found` when no errors could be found. + The `check` command now explicetly prints `No errors were found` when no errors + could be found. https://github.com/restic/restic/issues/1303 https://github.com/restic/restic/pull/1319 @@ -4845,8 +5134,8 @@ restic users. The changes are ordered by importance. * Enhancement #1367: Allow comments in files read from via `--file-from` - When the list of files/dirs to be saved is read from a file with `--files-from`, comment lines - (starting with `#`) are now ignored. + When the list of files/dirs to be saved is read from a file with `--files-from`, + comment lines (starting with `#`) are now ignored. https://github.com/restic/restic/issues/1367 https://github.com/restic/restic/pull/1368 @@ -4864,9 +5153,10 @@ restic users. The changes are ordered by importance. * Bugfix #1246: List all files stored in Google Cloud Storage - For large backups stored in Google Cloud Storage, the `prune` command fails because listing - only returns the first 1000 files. This has been corrected, no data is lost in the process. In - addition, a plausibility check was added to `prune`. + For large backups stored in Google Cloud Storage, the `prune` command fails + because listing only returns the first 1000 files. This has been corrected, no + data is lost in the process. In addition, a plausibility check was added to + `prune`. https://github.com/restic/restic/issues/1246 https://github.com/restic/restic/pull/1247 @@ -4904,26 +5194,28 @@ restic users. The changes are ordered by importance. * Bugfix #1167: Do not create a local repo unless `init` is used - When a restic command other than `init` is used with a local repository and the repository - directory does not exist, restic creates the directory structure. That's an error, only the - `init` command should create the dir. + When a restic command other than `init` is used with a local repository and the + repository directory does not exist, restic creates the directory structure. + That's an error, only the `init` command should create the dir. https://github.com/restic/restic/issues/1167 https://github.com/restic/restic/pull/1182 * Bugfix #1191: Make sure to write profiling files on interrupt - Since a few releases restic had the ability to write profiling files for memory and CPU usage - when `debug` is enabled. It was discovered that when restic is interrupted (ctrl+c is - pressed), the proper shutdown hook is not run. This is now corrected. + Since a few releases restic had the ability to write profiling files for memory + and CPU usage when `debug` is enabled. It was discovered that when restic is + interrupted (ctrl+c is pressed), the proper shutdown hook is not run. This is + now corrected. https://github.com/restic/restic/pull/1191 * Enhancement #317: Add `--exclude-caches` and `--exclude-if-present` - A new option `--exclude-caches` was added that allows excluding cache directories (that are - tagged as such). This is a special case of a more generic option `--exclude-if-present` which - excludes a directory if a file with a specific name (and contents) is present. + A new option `--exclude-caches` was added that allows excluding cache + directories (that are tagged as such). This is a special case of a more generic + option `--exclude-if-present` which excludes a directory if a file with a + specific name (and contents) is present. https://github.com/restic/restic/issues/317 https://github.com/restic/restic/pull/1170 @@ -4944,16 +5236,17 @@ restic users. The changes are ordered by importance. * Enhancement #1126: Use the standard Go git repository layout, use `dep` for vendoring - The git repository layout was changed to resemble the layout typically used in Go projects, - we're not using `gb` for building restic any more and vendoring the dependencies is now taken - care of by `dep`. + The git repository layout was changed to resemble the layout typically used in + Go projects, we're not using `gb` for building restic any more and vendoring the + dependencies is now taken care of by `dep`. https://github.com/restic/restic/pull/1126 * Enhancement #1132: Make `key` command always prompt for a password - The `key` command now prompts for a password even if the original password to access a repo has - been specified via the `RESTIC_PASSWORD` environment variable or a password file. + The `key` command now prompts for a password even if the original password to + access a repo has been specified via the `RESTIC_PASSWORD` environment variable + or a password file. https://github.com/restic/restic/issues/1132 https://github.com/restic/restic/pull/1133 @@ -4970,8 +5263,8 @@ restic users. The changes are ordered by importance. * Enhancement #1149: Add support for storing backups on Microsoft Azure Blob Storage - The library we're using to access the service requires Go 1.8, so restic now needs at least Go - 1.8. + The library we're using to access the service requires Go 1.8, so restic now + needs at least Go 1.8. https://github.com/restic/restic/issues/609 https://github.com/restic/restic/pull/1149 @@ -4997,8 +5290,8 @@ restic users. The changes are ordered by importance. * Enhancement #1218: Add `--compact` to `snapshots` command - The option `--compact` was added to the `snapshots` command to get a better overview of the - snapshots in a repo. It limits each snapshot to a single line. + The option `--compact` was added to the `snapshots` command to get a better + overview of the snapshots in a repo. It limits each snapshot to a single line. https://github.com/restic/restic/issues/1218 https://github.com/restic/restic/pull/1223 @@ -5022,18 +5315,19 @@ restic users. The changes are ordered by importance. * Bugfix #1115: Fix `prune`, only include existing files in indexes - A bug was found (and corrected) in the index rebuilding after prune, which led to indexes which - include blobs that were not present in the repo any more. There were already checks in place - which detected this situation and aborted with an error message. A new run of either `prune` or - `rebuild-index` corrected the index files. This is now fixed and a test has been added to detect - this. + A bug was found (and corrected) in the index rebuilding after prune, which led + to indexes which include blobs that were not present in the repo any more. There + were already checks in place which detected this situation and aborted with an + error message. A new run of either `prune` or `rebuild-index` corrected the + index files. This is now fixed and a test has been added to detect this. https://github.com/restic/restic/pull/1115 * Enhancement #1055: Create subdirs below `data/` for local/sftp backends - The local and sftp backends now create the subdirs below `data/` on open/init. This way, restic - makes sure that they always exist. This is connected to an issue for the sftp server. + The local and sftp backends now create the subdirs below `data/` on open/init. + This way, restic makes sure that they always exist. This is connected to an + issue for the sftp server. https://github.com/restic/restic/issues/1055 https://github.com/restic/rest-server/pull/11#issuecomment-309879710 @@ -5042,17 +5336,18 @@ restic users. The changes are ordered by importance. * Enhancement #1067: Allow loading credentials for s3 from IAM - When no S3 credentials are specified in the environment variables, restic now tries to load - credentials from an IAM instance profile when the s3 backend is used. + When no S3 credentials are specified in the environment variables, restic now + tries to load credentials from an IAM instance profile when the s3 backend is + used. https://github.com/restic/restic/issues/1067 https://github.com/restic/restic/pull/1086 * Enhancement #1073: Add `migrate` cmd to migrate from `s3legacy` to `default` layout - The `migrate` command for changing the `s3legacy` layout to the `default` layout for s3 - backends has been improved: It can now be restarted with `restic migrate --force s3_layout` - and automatically retries operations on error. + The `migrate` command for changing the `s3legacy` layout to the `default` layout + for s3 backends has been improved: It can now be restarted with `restic migrate + --force s3_layout` and automatically retries operations on error. https://github.com/restic/restic/issues/1073 https://github.com/restic/restic/pull/1075 @@ -5092,18 +5387,18 @@ restic users. The changes are ordered by importance. * Bugfix #965: Switch to `default` repo layout for the s3 backend - The default layout for the s3 backend is now `default` (instead of `s3legacy`). Also, there's a - new `migrate` command to convert an existing repo, it can be run like this: `restic migrate - s3_layout` + The default layout for the s3 backend is now `default` (instead of `s3legacy`). + Also, there's a new `migrate` command to convert an existing repo, it can be run + like this: `restic migrate s3_layout` https://github.com/restic/restic/issues/965 https://github.com/restic/restic/pull/1004 * Bugfix #1013: Switch back to using the high-level minio-go API for s3 - For the s3 backend we're back to using the high-level API the s3 client library for uploading - data, a few users reported dropped connections (which the library will automatically retry - now). + For the s3 backend we're back to using the high-level API the s3 client library + for uploading data, a few users reported dropped connections (which the library + will automatically retry now). https://github.com/restic/restic/issues/1013 https://github.com/restic/restic/issues/1023 @@ -5116,9 +5411,10 @@ restic users. The changes are ordered by importance. * Enhancement #636: Add dirs `tags` and `hosts` to fuse mount - The fuse mount now has two more directories: `tags` contains a subdir for each tag, which in turn - contains only the snapshots that have this tag. The subdir `hosts` contains a subdir for each - host that has a snapshot, and the subdir contains the snapshots for that host. + The fuse mount now has two more directories: `tags` contains a subdir for each + tag, which in turn contains only the snapshots that have this tag. The subdir + `hosts` contains a subdir for each host that has a snapshot, and the subdir + contains the snapshots for that host. https://github.com/restic/restic/issues/636 https://github.com/restic/restic/pull/1050 @@ -5130,8 +5426,9 @@ restic users. The changes are ordered by importance. * Enhancement #989: Improve performance of the `find` command - Improved performance for the `find` command: Restic recognizes paths it has already checked - for the files in question, so the number of backend requests is reduced a lot. + Improved performance for the `find` command: Restic recognizes paths it has + already checked for the files in question, so the number of backend requests is + reduced a lot. https://github.com/restic/restic/issues/989 https://github.com/restic/restic/pull/993 @@ -5144,16 +5441,17 @@ restic users. The changes are ordered by importance. * Enhancement #1021: Detect invalid backend name and print error - Restic now tries to detect when an invalid/unknown backend is used and returns an error - message. + Restic now tries to detect when an invalid/unknown backend is used and returns + an error message. https://github.com/restic/restic/issues/1021 https://github.com/restic/restic/pull/1070 * Enhancement #1029: Remove invalid pack files when `prune` is run - The `prune` command has been improved and will now remove invalid pack files, for example files - that have not been uploaded completely because a backup was interrupted. + The `prune` command has been improved and will now remove invalid pack files, + for example files that have not been uploaded completely because a backup was + interrupted. https://github.com/restic/restic/issues/1029 https://github.com/restic/restic/pull/1036 @@ -5173,24 +5471,24 @@ restic users. The changes are ordered by importance. * Enhancement #974: Remove regular status reports - Regular status report: We've removed the status report that was printed every 10 seconds when - restic is run non-interactively. You can still force reporting the current status by sending a - `USR1` signal to the process. + Regular status report: We've removed the status report that was printed every 10 + seconds when restic is run non-interactively. You can still force reporting the + current status by sending a `USR1` signal to the process. https://github.com/restic/restic/pull/974 * Enhancement #981: Remove temporary path from binary in `build.go` - The `build.go` now strips the temporary directory used for compilation from the binary. This - is the first step in enabling reproducible builds. + The `build.go` now strips the temporary directory used for compilation from the + binary. This is the first step in enabling reproducible builds. https://github.com/restic/restic/pull/981 * Enhancement #985: Allow multiple parallel idle HTTP connections - Backends based on HTTP now allow several idle connections in parallel. This is especially - important for the REST backend, which (when used with a local server) may create a lot - connections and exhaust available ports quickly. + Backends based on HTTP now allow several idle connections in parallel. This is + especially important for the REST backend, which (when used with a local server) + may create a lot connections and exhaust available ports quickly. https://github.com/restic/restic/issues/985 https://github.com/restic/restic/pull/986 @@ -5210,21 +5508,22 @@ restic users. The changes are ordered by importance. * Enhancement #957: Make `forget` consistent - The `forget` command was corrected to be more consistent in which snapshots are to be - forgotten. It is possible that the new code removes more snapshots than before, so please - review what would be deleted by using the `--dry-run` option. + The `forget` command was corrected to be more consistent in which snapshots are + to be forgotten. It is possible that the new code removes more snapshots than + before, so please review what would be deleted by using the `--dry-run` option. https://github.com/restic/restic/issues/953 https://github.com/restic/restic/pull/957 * Enhancement #962: Improve memory and runtime for the s3 backend - We've updated the library used for accessing s3, switched to using a lower level API and added - caching for some requests. This lead to a decrease in memory usage and a great speedup. In - addition, we added benchmark functions for all backends, so we can track improvements over - time. The Continuous Integration test service we're using (Travis) now runs the s3 backend - tests not only against a Minio server, but also against the Amazon s3 live service, so we should - be notified of any regressions much sooner. + We've updated the library used for accessing s3, switched to using a lower level + API and added caching for some requests. This lead to a decrease in memory usage + and a great speedup. In addition, we added benchmark functions for all backends, + so we can track improvements over time. The Continuous Integration test service + we're using (Travis) now runs the s3 backend tests not only against a Minio + server, but also against the Amazon s3 live service, so we should be notified of + any regressions much sooner. https://github.com/restic/restic/pull/962 https://github.com/restic/restic/pull/960 @@ -5234,11 +5533,12 @@ restic users. The changes are ordered by importance. * Enhancement #966: Unify repository layout for all backends - Up to now the s3 backend used a special repository layout. We've decided to unify the repository - layout and implemented the default layout also for the s3 backend. For creating a new - repository on s3 with the default layout, use `restic -o s3.layout=default init`. For further - commands the option is not necessary any more, restic will automatically detect the correct - layout to use. A future version will switch to the default layout for new repositories. + Up to now the s3 backend used a special repository layout. We've decided to + unify the repository layout and implemented the default layout also for the s3 + backend. For creating a new repository on s3 with the default layout, use + `restic -o s3.layout=default init`. For further commands the option is not + necessary any more, restic will automatically detect the correct layout to use. + A future version will switch to the default layout for new repositories. https://github.com/restic/restic/issues/965 https://github.com/restic/restic/pull/966 From abca1124041f9c83227d253142dcc713e7b3914c Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 14 Jan 2024 20:21:45 +0100 Subject: [PATCH 031/158] Add version for 0.16.3 --- VERSION | 2 +- cmd/restic/global.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 201a22c8f..7eb3095a3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.16.2 +0.16.3 diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 8101354ce..c11aca615 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -43,7 +43,7 @@ import ( "golang.org/x/term" ) -var version = "0.16.2-dev (compiled manually)" +var version = "0.16.3" // TimeFormat is the format used for all timestamps printed by restic. const TimeFormat = "2006-01-02 15:04:05" From 0ec9383ba2da1ca6020ac8fc757c9ccfd22a9cac Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 14 Jan 2024 20:21:45 +0100 Subject: [PATCH 032/158] Set development version for 0.16.3 --- cmd/restic/global.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index c11aca615..9e3a3a7a9 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -43,7 +43,7 @@ import ( "golang.org/x/term" ) -var version = "0.16.3" +var version = "0.16.3-dev (compiled manually)" // TimeFormat is the format used for all timestamps printed by restic. const TimeFormat = "2006-01-02 15:04:05" From 5413877d3358fba9bdd6d9b2c266308c57f862f1 Mon Sep 17 00:00:00 2001 From: Ben Northway Date: Thu, 18 Jan 2024 16:33:57 -0600 Subject: [PATCH 033/158] Update Backblaze documentation clarify documentation regarding B2 bucket lifecycle settings. The default lifecycle setting is probably fine for most users now; a custom policy is not necessary. --- doc/030_preparing_a_new_repo.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index 5ff26934a..40215ad81 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -487,7 +487,8 @@ Backblaze B2 Different from the B2 backend, restic's S3 backend will only hide no longer necessary files. Thus, make sure to setup lifecycle rules to eventually - delete hidden files. + delete hidden files. The lifecycle setting "Keep only the last version of the file" + will keep only the most current version of a file. Read the [Backblaze documentation](https://www.backblaze.com/docs/cloud-storage-lifecycle-rules). Restic can backup data to any Backblaze B2 bucket. You need to first setup the following environment variables with the credentials you can find in the From 0b39940fdb8b39a138a3d58764efa80f10c94838 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 13:59:47 +0100 Subject: [PATCH 034/158] walker: Remove ignoreTrees functionality It was only used in two places: - stats: apparently as a minor performance optimization, which is unlikely to be important - find: filtered directories would be ignored. However, this optimization missed that it is possible that two directories have the exact same content. Such directories would be incorrectly ignored too. Example: ``` mkdir test test/a test/b restic backup test restic find latest test/b -> incorrectly does not return anything ``` Thus, remove the functionality as it's apparently too complex to use correctly. --- changelog/unreleased/pull-4615 | 6 ++ cmd/restic/cmd_find.go | 56 +++++----- cmd/restic/cmd_ls.go | 14 +-- cmd/restic/cmd_stats.go | 14 ++- internal/dump/common.go | 12 +-- internal/walker/walker.go | 74 +++----------- internal/walker/walker_test.go | 182 +++------------------------------ 7 files changed, 77 insertions(+), 281 deletions(-) create mode 100644 changelog/unreleased/pull-4615 diff --git a/changelog/unreleased/pull-4615 b/changelog/unreleased/pull-4615 new file mode 100644 index 000000000..7e2d4a017 --- /dev/null +++ b/changelog/unreleased/pull-4615 @@ -0,0 +1,6 @@ +Bugfix: `find` ignored directories in some cases + +In some cases, the `find` command ignored empty or moved directories. This has +been fixed. + +https://github.com/restic/restic/pull/4615 diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index c30650823..33fff864f 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -244,13 +244,12 @@ func (s *statefulOutput) Finish() { // Finder bundles information needed to find a file or directory. type Finder struct { - repo restic.Repository - pat findPattern - out statefulOutput - ignoreTrees restic.IDSet - blobIDs map[string]struct{} - treeIDs map[string]struct{} - itemsFound int + repo restic.Repository + pat findPattern + out statefulOutput + blobIDs map[string]struct{} + treeIDs map[string]struct{} + itemsFound int } func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error { @@ -261,17 +260,17 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error } f.out.newsn = sn - return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { + return walker.Walk(ctx, f.repo, *sn.Tree, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { debug.Log("Error loading tree %v: %v", parentTreeID, err) Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID()) - return false, walker.ErrSkipNode + return walker.ErrSkipNode } if node == nil { - return false, nil + return nil } normalizedNodepath := nodepath @@ -284,7 +283,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error for _, pat := range f.pat.pattern { found, err := filter.Match(pat, normalizedNodepath) if err != nil { - return false, err + return err } if found { foundMatch = true @@ -292,16 +291,13 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error } } - var ( - ignoreIfNoMatch = true - errIfNoMatch error - ) + var errIfNoMatch error if node.Type == "dir" { var childMayMatch bool for _, pat := range f.pat.pattern { mayMatch, err := filter.ChildMatch(pat, normalizedNodepath) if err != nil { - return false, err + return err } if mayMatch { childMayMatch = true @@ -310,30 +306,27 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error } if !childMayMatch { - ignoreIfNoMatch = true errIfNoMatch = walker.ErrSkipNode - } else { - ignoreIfNoMatch = false } } if !foundMatch { - return ignoreIfNoMatch, errIfNoMatch + return errIfNoMatch } if !f.pat.oldest.IsZero() && node.ModTime.Before(f.pat.oldest) { debug.Log(" ModTime is older than %s\n", f.pat.oldest) - return ignoreIfNoMatch, errIfNoMatch + return errIfNoMatch } if !f.pat.newest.IsZero() && node.ModTime.After(f.pat.newest) { debug.Log(" ModTime is newer than %s\n", f.pat.newest) - return ignoreIfNoMatch, errIfNoMatch + return errIfNoMatch } debug.Log(" found match\n") f.out.PrintPattern(nodepath, node) - return false, nil + return nil }) } @@ -345,17 +338,17 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { } f.out.newsn = sn - return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { + return walker.Walk(ctx, f.repo, *sn.Tree, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { debug.Log("Error loading tree %v: %v", parentTreeID, err) Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID()) - return false, walker.ErrSkipNode + return walker.ErrSkipNode } if node == nil { - return false, nil + return nil } if node.Type == "dir" && f.treeIDs != nil { @@ -373,7 +366,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { // looking for blobs) if f.itemsFound >= len(f.treeIDs) && f.blobIDs == nil { // Return an error to terminate the Walk - return true, errors.New("OK") + return errors.New("OK") } } } @@ -394,7 +387,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { } } - return false, nil + return nil }) } @@ -593,10 +586,9 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args [] } f := &Finder{ - repo: repo, - pat: pat, - out: statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON}, - ignoreTrees: restic.NewIDSet(), + repo: repo, + pat: pat, + out: statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON}, } if opts.BlobID { diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 83a03559d..07c49d60f 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -230,12 +230,12 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri printSnapshot(sn) - err = walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { + err = walker.Walk(ctx, repo, *sn.Tree, func(_ restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { - return false, err + return err } if node == nil { - return false, nil + return nil } if withinDir(nodepath) { @@ -245,22 +245,22 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri // if recursive listing is requested, signal the walker that it // should continue walking recursively if opts.Recursive { - return false, nil + return nil } } // if there's an upcoming match deeper in the tree (but we're not // there yet), signal the walker to descend into any subdirs if approachingMatchingTree(nodepath) { - return false, nil + return nil } // otherwise, signal the walker to not walk recursively into any // subdirs if node.Type == "dir" { - return false, walker.ErrSkipNode + return walker.ErrSkipNode } - return false, nil + return nil }) if err != nil { diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index a3e0cefc7..b0837510d 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -203,7 +203,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest } hardLinkIndex := restorer.NewHardlinkIndex[struct{}]() - err := walker.Walk(ctx, repo, *snapshot.Tree, restic.NewIDSet(), statsWalkTree(repo, opts, stats, hardLinkIndex)) + err := walker.Walk(ctx, repo, *snapshot.Tree, statsWalkTree(repo, opts, stats, hardLinkIndex)) if err != nil { return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err) } @@ -212,12 +212,12 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest } func statsWalkTree(repo restic.Repository, opts StatsOptions, stats *statsContainer, hardLinkIndex *restorer.HardlinkIndex[struct{}]) walker.WalkFunc { - return func(parentTreeID restic.ID, npath string, node *restic.Node, nodeErr error) (bool, error) { + return func(parentTreeID restic.ID, npath string, node *restic.Node, nodeErr error) error { if nodeErr != nil { - return true, nodeErr + return nodeErr } if node == nil { - return true, nil + return nil } if opts.countMode == countModeUniqueFilesByContents || opts.countMode == countModeBlobsPerFile { @@ -247,7 +247,7 @@ func statsWalkTree(repo restic.Repository, opts StatsOptions, stats *statsContai // is always a data blob since we're accessing it via a file's Content array blobSize, found := repo.LookupBlobSize(blobID, restic.DataBlob) if !found { - return true, fmt.Errorf("blob %s not found for tree %s", blobID, parentTreeID) + return fmt.Errorf("blob %s not found for tree %s", blobID, parentTreeID) } // count the blob's size, then add this blob by this @@ -274,11 +274,9 @@ func statsWalkTree(repo restic.Repository, opts StatsOptions, stats *statsContai hardLinkIndex.Add(node.Inode, node.DeviceID, struct{}{}) stats.TotalSize += node.Size } - - return false, nil } - return true, nil + return nil } } diff --git a/internal/dump/common.go b/internal/dump/common.go index c3ba69431..3ca1ced82 100644 --- a/internal/dump/common.go +++ b/internal/dump/common.go @@ -70,27 +70,27 @@ func sendNodes(ctx context.Context, repo restic.Repository, root *restic.Node, c return nil } - err := walker.Walk(ctx, repo, *root.Subtree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) { + err := walker.Walk(ctx, repo, *root.Subtree, func(_ restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { - return false, err + return err } if node == nil { - return false, nil + return nil } node.Path = path.Join(root.Path, nodepath) if !IsFile(node) && !IsDir(node) && !IsLink(node) { - return false, nil + return nil } select { case ch <- node: case <-ctx.Done(): - return false, ctx.Err() + return ctx.Err() } - return false, nil + return nil }) return err diff --git a/internal/walker/walker.go b/internal/walker/walker.go index 4c4e7f5ab..aba2e39e5 100644 --- a/internal/walker/walker.go +++ b/internal/walker/walker.go @@ -21,21 +21,14 @@ var ErrSkipNode = errors.New("skip this node") // When the special value ErrSkipNode is returned and node is a dir node, it is // not walked. When the node is not a dir node, the remaining items in this // tree are skipped. -// -// Setting ignore to true tells Walk that it should not visit the node again. -// For tree nodes, this means that the function is not called for the -// referenced tree. If the node is not a tree, and all nodes in the current -// tree have ignore set to true, the current tree will not be visited again. -// When err is not nil and different from ErrSkipNode, the value returned for -// ignore is ignored. -type WalkFunc func(parentTreeID restic.ID, path string, node *restic.Node, nodeErr error) (ignore bool, err error) +type WalkFunc func(parentTreeID restic.ID, path string, node *restic.Node, nodeErr error) (err error) // Walk calls walkFn recursively for each node in root. If walkFn returns an // error, it is passed up the call stack. The trees in ignoreTrees are not // walked. If walkFn ignores trees, these are added to the set. -func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, ignoreTrees restic.IDSet, walkFn WalkFunc) error { +func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, walkFn WalkFunc) error { tree, err := restic.LoadTree(ctx, repo, root) - _, err = walkFn(root, "/", nil, err) + err = walkFn(root, "/", nil, err) if err != nil { if err == ErrSkipNode { @@ -44,24 +37,13 @@ func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, ignoreTre return err } - if ignoreTrees == nil { - ignoreTrees = restic.NewIDSet() - } - - _, err = walk(ctx, repo, "/", root, tree, ignoreTrees, walkFn) - return err + return walk(ctx, repo, "/", root, tree, walkFn) } // walk recursively traverses the tree, ignoring subtrees when the ID of the // subtree is in ignoreTrees. If err is nil and ignore is true, the subtree ID // will be added to ignoreTrees by walk. -func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *restic.Tree, ignoreTrees restic.IDSet, walkFn WalkFunc) (ignore bool, err error) { - var allNodesIgnored = true - - if len(tree.Nodes) == 0 { - allNodesIgnored = false - } - +func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *restic.Tree, walkFn WalkFunc) (err error) { sort.Slice(tree.Nodes, func(i, j int) bool { return tree.Nodes[i].Name < tree.Nodes[j].Name }) @@ -70,68 +52,40 @@ func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTree p := path.Join(prefix, node.Name) if node.Type == "" { - return false, errors.Errorf("node type is empty for node %q", node.Name) + return errors.Errorf("node type is empty for node %q", node.Name) } if node.Type != "dir" { - ignore, err := walkFn(parentTreeID, p, node, nil) + err := walkFn(parentTreeID, p, node, nil) if err != nil { if err == ErrSkipNode { // skip the remaining entries in this tree - return allNodesIgnored, nil + return nil } - return false, err - } - - if !ignore { - allNodesIgnored = false + return err } continue } if node.Subtree == nil { - return false, errors.Errorf("subtree for node %v in tree %v is nil", node.Name, p) - } - - if ignoreTrees.Has(*node.Subtree) { - continue + return errors.Errorf("subtree for node %v in tree %v is nil", node.Name, p) } subtree, err := restic.LoadTree(ctx, repo, *node.Subtree) - ignore, err := walkFn(parentTreeID, p, node, err) + err = walkFn(parentTreeID, p, node, err) if err != nil { if err == ErrSkipNode { - if ignore { - ignoreTrees.Insert(*node.Subtree) - } continue } - return false, err } - if ignore { - ignoreTrees.Insert(*node.Subtree) - } - - if !ignore { - allNodesIgnored = false - } - - ignore, err = walk(ctx, repo, p, *node.Subtree, subtree, ignoreTrees, walkFn) + err = walk(ctx, repo, p, *node.Subtree, subtree, walkFn) if err != nil { - return false, err - } - - if ignore { - ignoreTrees.Insert(*node.Subtree) - } - - if !ignore { - allNodesIgnored = false + return err } } - return allNodesIgnored, nil + return nil } diff --git a/internal/walker/walker_test.go b/internal/walker/walker_test.go index 54cc69792..cf9dd4168 100644 --- a/internal/walker/walker_test.go +++ b/internal/walker/walker_test.go @@ -99,22 +99,22 @@ type checkFunc func(t testing.TB) (walker WalkFunc, final func(testing.TB)) func checkItemOrder(want []string) checkFunc { pos := 0 return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { - walker = func(treeID restic.ID, path string, node *restic.Node, err error) (bool, error) { + walker = func(treeID restic.ID, path string, node *restic.Node, err error) error { if err != nil { t.Errorf("error walking %v: %v", path, err) - return false, err + return err } if pos >= len(want) { t.Errorf("additional unexpected path found: %v", path) - return false, nil + return nil } if path != want[pos] { t.Errorf("wrong path found, want %q, got %q", want[pos], path) } pos++ - return false, nil + return nil } final = func(t testing.TB) { @@ -131,22 +131,22 @@ func checkItemOrder(want []string) checkFunc { func checkParentTreeOrder(want []string) checkFunc { pos := 0 return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { - walker = func(treeID restic.ID, path string, node *restic.Node, err error) (bool, error) { + walker = func(treeID restic.ID, path string, node *restic.Node, err error) error { if err != nil { t.Errorf("error walking %v: %v", path, err) - return false, err + return err } if pos >= len(want) { t.Errorf("additional unexpected parent tree ID found: %v", treeID) - return false, nil + return nil } if treeID.String() != want[pos] { t.Errorf("wrong parent tree ID found, want %q, got %q", want[pos], treeID.String()) } pos++ - return false, nil + return nil } final = func(t testing.TB) { @@ -165,15 +165,15 @@ func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc { var pos int return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { - walker = func(treeID restic.ID, path string, node *restic.Node, err error) (bool, error) { + walker = func(treeID restic.ID, path string, node *restic.Node, err error) error { if err != nil { t.Errorf("error walking %v: %v", path, err) - return false, err + return err } if pos >= len(wantPaths) { t.Errorf("additional unexpected path found: %v", path) - return false, nil + return nil } if path != wantPaths[pos] { @@ -182,50 +182,10 @@ func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc { pos++ if _, ok := skipFor[path]; ok { - return false, ErrSkipNode + return ErrSkipNode } - return false, nil - } - - final = func(t testing.TB) { - if pos != len(wantPaths) { - t.Errorf("wrong number of paths returned, want %d, got %d", len(wantPaths), pos) - } - } - - return walker, final - } -} - -// checkIgnore returns ErrSkipNode if path is in skipFor and sets ignore according -// to ignoreFor. It checks that the paths the walk func is called for are exactly -// the ones in wantPaths. -func checkIgnore(skipFor map[string]struct{}, ignoreFor map[string]bool, wantPaths []string) checkFunc { - var pos int - - return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { - walker = func(treeID restic.ID, path string, node *restic.Node, err error) (bool, error) { - if err != nil { - t.Errorf("error walking %v: %v", path, err) - return false, err - } - - if pos >= len(wantPaths) { - t.Errorf("additional unexpected path found: %v", path) - return ignoreFor[path], nil - } - - if path != wantPaths[pos] { - t.Errorf("wrong path found, want %q, got %q", wantPaths[pos], path) - } - pos++ - - if _, ok := skipFor[path]; ok { - return ignoreFor[path], ErrSkipNode - } - - return ignoreFor[path], nil + return nil } final = func(t testing.TB) { @@ -272,16 +232,6 @@ func TestWalker(t *testing.T) { "/subdir", }, ), - checkIgnore( - map[string]struct{}{}, map[string]bool{ - "/subdir": true, - }, []string{ - "/", - "/foo", - "/subdir", - "/subdir/subfile", - }, - ), }, }, { @@ -409,81 +359,6 @@ func TestWalker(t *testing.T) { "57ee8960c7a86859b090a76e5d013f83d10c0ce11d5460076ca8468706f784ab", // tree /subdir3 "c2efeff7f217a4dfa12a16e8bb3cefedd37c00873605c29e5271c6061030672f", // tree / }), - checkIgnore( - map[string]struct{}{ - "/subdir1": {}, - }, map[string]bool{ - "/subdir1": true, - }, []string{ - "/", - "/foo", - "/subdir1", - "/zzz other", - }, - ), - checkIgnore( - map[string]struct{}{}, map[string]bool{ - "/subdir1": true, - }, []string{ - "/", - "/foo", - "/subdir1", - "/subdir1/subfile1", - "/subdir1/subfile2", - "/subdir1/subfile3", - "/zzz other", - }, - ), - checkIgnore( - map[string]struct{}{ - "/subdir2": {}, - }, map[string]bool{ - "/subdir2": true, - }, []string{ - "/", - "/foo", - "/subdir1", - "/subdir1/subfile1", - "/subdir1/subfile2", - "/subdir1/subfile3", - "/subdir2", - "/zzz other", - }, - ), - checkIgnore( - map[string]struct{}{}, map[string]bool{ - "/subdir1/subfile1": true, - "/subdir1/subfile2": true, - "/subdir1/subfile3": true, - }, []string{ - "/", - "/foo", - "/subdir1", - "/subdir1/subfile1", - "/subdir1/subfile2", - "/subdir1/subfile3", - "/zzz other", - }, - ), - checkIgnore( - map[string]struct{}{}, map[string]bool{ - "/subdir2/subfile1": true, - "/subdir2/subfile2": true, - "/subdir2/subfile3": true, - }, []string{ - "/", - "/foo", - "/subdir1", - "/subdir1/subfile1", - "/subdir1/subfile2", - "/subdir1/subfile3", - "/subdir2", - "/subdir2/subfile1", - "/subdir2/subfile2", - "/subdir2/subfile3", - "/zzz other", - }, - ), }, }, { @@ -513,35 +388,6 @@ func TestWalker(t *testing.T) { }), }, }, - { - tree: TestTree{ - "subdir1": TestTree{}, - "subdir2": TestTree{}, - "subdir3": TestTree{ - "file": TestFile{}, - }, - "subdir4": TestTree{}, - "subdir5": TestTree{ - "file": TestFile{}, - }, - "subdir6": TestTree{}, - }, - checks: []checkFunc{ - checkIgnore( - map[string]struct{}{}, map[string]bool{ - "/subdir2": true, - }, []string{ - "/", - "/subdir1", - "/subdir2", - "/subdir3", - "/subdir3/file", - "/subdir5", - "/subdir5/file", - }, - ), - }, - }, } for _, test := range tests { @@ -553,7 +399,7 @@ func TestWalker(t *testing.T) { defer cancel() fn, last := check(t) - err := Walk(ctx, repo, root, restic.NewIDSet(), fn) + err := Walk(ctx, repo, root, fn) if err != nil { t.Error(err) } From fdcbb53017438bbcc572d7af021899b33d6a59c7 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 19 Jan 2024 21:14:31 +0100 Subject: [PATCH 035/158] walker: test skipping for root node --- internal/walker/walker_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/walker/walker_test.go b/internal/walker/walker_test.go index cf9dd4168..786570e02 100644 --- a/internal/walker/walker_test.go +++ b/internal/walker/walker_test.go @@ -232,6 +232,13 @@ func TestWalker(t *testing.T) { "/subdir", }, ), + checkSkipFor( + map[string]struct{}{ + "/": {}, + }, []string{ + "/", + }, + ), }, }, { From b1a8fd1d03928673e024f96e7fc9cf62fcbd90b2 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 15:24:33 +0100 Subject: [PATCH 036/158] rest: fix and cleanup closing of http response body If client.Do returns an error, then there's no body that has to be closed. For requests for which we are not interested in the response body, immediately drain and close the body to make sure it isn't forgotten later on. This change in particular adds the missing `Close()` call for the `List()` command. --- internal/backend/rest/rest.go | 79 +++++++++++++++++------------------ 1 file changed, 38 insertions(+), 41 deletions(-) diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index 5310eba7c..d8171d90e 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -58,6 +58,17 @@ func Open(_ context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) return be, nil } +func drainAndClose(resp *http.Response) error { + _, err := io.Copy(io.Discard, resp.Body) + cerr := resp.Body.Close() + + // return first error + if err != nil { + return errors.Errorf("drain: %w", err) + } + return cerr +} + // Create creates a new REST on server configured in config. func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, error) { be, err := Open(ctx, cfg, rt) @@ -80,20 +91,14 @@ func Create(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, er return nil, err } + if err := drainAndClose(resp); err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("server response unexpected: %v (%v)", resp.Status, resp.StatusCode) } - _, err = io.Copy(io.Discard, resp.Body) - if err != nil { - return nil, err - } - - err = resp.Body.Close() - if err != nil { - return nil, err - } - return be, nil } @@ -136,22 +141,19 @@ func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindR req.ContentLength = rd.Length() resp, err := b.client.Do(req) - - var cerr error - if resp != nil { - _, _ = io.Copy(io.Discard, resp.Body) - cerr = resp.Body.Close() - } - if err != nil { return errors.WithStack(err) } + if err := drainAndClose(resp); err != nil { + return err + } + if resp.StatusCode != http.StatusOK { return errors.Errorf("server response unexpected: %v (%v)", resp.Status, resp.StatusCode) } - return errors.Wrap(cerr, "Close") + return nil } // notExistError is returned whenever the requested file does not exist on the @@ -215,22 +217,17 @@ func (b *Backend) openReader(ctx context.Context, h backend.Handle, length int, req.Header.Set("Accept", ContentTypeV2) resp, err := b.client.Do(req) - if err != nil { - if resp != nil { - _, _ = io.Copy(io.Discard, resp.Body) - _ = resp.Body.Close() - } return nil, errors.Wrap(err, "client.Do") } if resp.StatusCode == http.StatusNotFound { - _ = resp.Body.Close() + _ = drainAndClose(resp) return nil, ¬ExistError{h} } if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { - _ = resp.Body.Close() + _ = drainAndClose(resp) return nil, errors.Errorf("unexpected HTTP response (%v): %v", resp.StatusCode, resp.Status) } @@ -250,13 +247,11 @@ func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, return backend.FileInfo{}, errors.WithStack(err) } - _, _ = io.Copy(io.Discard, resp.Body) - if err = resp.Body.Close(); err != nil { - return backend.FileInfo{}, errors.Wrap(err, "Close") + if err = drainAndClose(resp); err != nil { + return backend.FileInfo{}, err } if resp.StatusCode == http.StatusNotFound { - _ = resp.Body.Close() return backend.FileInfo{}, ¬ExistError{h} } @@ -285,13 +280,15 @@ func (b *Backend) Remove(ctx context.Context, h backend.Handle) error { req.Header.Set("Accept", ContentTypeV2) resp, err := b.client.Do(req) - if err != nil { return errors.Wrap(err, "client.Do") } + if err = drainAndClose(resp); err != nil { + return err + } + if resp.StatusCode == http.StatusNotFound { - _ = resp.Body.Close() return ¬ExistError{h} } @@ -299,12 +296,7 @@ func (b *Backend) Remove(ctx context.Context, h backend.Handle) error { return errors.Errorf("blob not removed, server response: %v (%v)", resp.Status, resp.StatusCode) } - _, err = io.Copy(io.Discard, resp.Body) - if err != nil { - return errors.Wrap(err, "Copy") - } - - return errors.Wrap(resp.Body.Close(), "Close") + return nil } // List runs fn for each file in the backend which has the type t. When an @@ -322,7 +314,6 @@ func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(backend. req.Header.Set("Accept", ContentTypeV2) resp, err := b.client.Do(req) - if err != nil { return errors.Wrap(err, "List") } @@ -333,19 +324,25 @@ func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(backend. // already ignores missing directories, but misuses "not found" to // report certain internal errors, see // https://github.com/rclone/rclone/pull/7550 for details. - return nil + return drainAndClose(resp) } } if resp.StatusCode != http.StatusOK { + _ = drainAndClose(resp) return errors.Errorf("List failed, server response: %v (%v)", resp.Status, resp.StatusCode) } if resp.Header.Get("Content-Type") == ContentTypeV2 { - return b.listv2(ctx, resp, fn) + err = b.listv2(ctx, resp, fn) + } else { + err = b.listv1(ctx, t, resp, fn) } - return b.listv1(ctx, t, resp, fn) + if cerr := drainAndClose(resp); cerr != nil && err == nil { + err = cerr + } + return err } // listv1 uses the REST protocol v1, where a list HTTP request (e.g. `GET From bd883caae1333b99f7cec82b51c6945410cde83e Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 15:38:57 +0100 Subject: [PATCH 037/158] CI: enable bodyclose linter --- .golangci.yml | 3 +++ internal/backend/limiter/static_limiter_test.go | 1 + internal/backend/rclone/backend.go | 1 + 3 files changed, 5 insertions(+) diff --git a/.golangci.yml b/.golangci.yml index 98b5f9e03..c08331401 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -35,6 +35,9 @@ linters: # parse and typecheck code - typecheck + # ensure that http response bodies are closed + - bodyclose + issues: # don't use the default exclude rules, this hides (among others) ignored # errors from Close() calls diff --git a/internal/backend/limiter/static_limiter_test.go b/internal/backend/limiter/static_limiter_test.go index 8a839518f..79a1d02f3 100644 --- a/internal/backend/limiter/static_limiter_test.go +++ b/internal/backend/limiter/static_limiter_test.go @@ -118,6 +118,7 @@ func TestRoundTripperReader(t *testing.T) { test.Assert(t, bytes.Equal(data, out.Bytes()), "data ping-pong failed") } +// nolint:bodyclose // the http response is just a mock func TestRoundTripperCornerCases(t *testing.T) { limiter := NewStaticLimiter(Limits{42 * 1024, 42 * 1024}) diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go index a41a89898..416162364 100644 --- a/internal/backend/rclone/backend.go +++ b/internal/backend/rclone/backend.go @@ -252,6 +252,7 @@ func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter) (*Backend, return nil, fmt.Errorf("error talking HTTP to rclone: %w", err) } + _ = res.Body.Close() debug.Log("HTTP status %q returned, moving instance to background", res.Status) err = bg() if err != nil { From fb422497af69c1267f242275e817257424e08be4 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 31 Dec 2023 00:18:41 +0100 Subject: [PATCH 038/158] repository: split StreamPack implementation Move the actual decoding of the pack data into a separate iterator. --- internal/repository/repository.go | 173 +++++++++++++++++++----------- 1 file changed, 111 insertions(+), 62 deletions(-) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 97dc33fdf..e13220741 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -882,7 +882,7 @@ const maxUnusedRange = 4 * 1024 * 1024 // StreamPack loads the listed blobs from the specified pack file. The plaintext blob is passed to // the handleBlobFn callback or an error if decryption failed or the blob hash does not match. -// handleBlobFn is never called multiple times for the same blob. If the callback returns an error, +// handleBlobFn is called at most once for each blob. If the callback returns an error, // then StreamPack will abort and not retry it. func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { if len(blobs) == 0 { @@ -940,72 +940,18 @@ func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, if bufferSize > MaxStreamBufferSize { bufferSize = MaxStreamBufferSize } - // create reader here to allow reusing the buffered reader from checker.checkData bufRd := bufio.NewReaderSize(rd, bufferSize) - currentBlobEnd := dataStart - var buf []byte - var decode []byte - for len(blobs) > 0 { - entry := blobs[0] + it := NewPackBlobIterator(packID, bufRd, dataStart, blobs, key, dec) - skipBytes := int(entry.Offset - currentBlobEnd) - if skipBytes < 0 { - return errors.Errorf("overlapping blobs in pack %v", packID) - } - - _, err := bufRd.Discard(skipBytes) - if err != nil { + for { + val, err := it.Next() + if err == ErrPackEOF { + break + } else if err != nil { return err } - h := restic.BlobHandle{ID: entry.ID, Type: entry.Type} - debug.Log(" process blob %v, skipped %d, %v", h, skipBytes, entry) - - if uint(cap(buf)) < entry.Length { - buf = make([]byte, entry.Length) - } - buf = buf[:entry.Length] - - n, err := io.ReadFull(bufRd, buf) - if err != nil { - debug.Log(" read error %v", err) - return errors.Wrap(err, "ReadFull") - } - - if n != len(buf) { - return errors.Errorf("read blob %v from %v: not enough bytes read, want %v, got %v", - h, packID.Str(), len(buf), n) - } - currentBlobEnd = entry.Offset + entry.Length - - if int(entry.Length) <= key.NonceSize() { - debug.Log("%v", blobs) - return errors.Errorf("invalid blob length %v", entry) - } - - // decryption errors are likely permanent, give the caller a chance to skip them - nonce, ciphertext := buf[:key.NonceSize()], buf[key.NonceSize():] - plaintext, err := key.Open(ciphertext[:0], nonce, ciphertext, nil) - if err == nil && entry.IsCompressed() { - // DecodeAll will allocate a slice if it is not large enough since it - // knows the decompressed size (because we're using EncodeAll) - decode, err = dec.DecodeAll(plaintext, decode[:0]) - plaintext = decode - if err != nil { - err = errors.Errorf("decompressing blob %v failed: %v", h, err) - } - } - if err == nil { - id := restic.Hash(plaintext) - if !id.Equal(entry.ID) { - debug.Log("read blob %v/%v from %v: wrong data returned, hash is %v", - h.Type, h.ID, packID.Str(), id) - err = errors.Errorf("read blob %v from %v: wrong data returned, hash is %v", - h, packID.Str(), id) - } - } - - err = handleBlobFn(entry.BlobHandle, plaintext, err) + err = handleBlobFn(val.Handle, val.Plaintext, val.Err) if err != nil { cancel() return backoff.Permanent(err) @@ -1018,6 +964,109 @@ func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, return errors.Wrap(err, "StreamPack") } +type PackBlobIterator struct { + packID restic.ID + rd *bufio.Reader + currentOffset uint + + blobs []restic.Blob + key *crypto.Key + dec *zstd.Decoder + + buf []byte + decode []byte +} + +type PackBlobValue struct { + Handle restic.BlobHandle + Plaintext []byte + Err error +} + +var ErrPackEOF = errors.New("reached EOF of pack file") + +func NewPackBlobIterator(packID restic.ID, rd *bufio.Reader, currentOffset uint, + blobs []restic.Blob, key *crypto.Key, dec *zstd.Decoder) *PackBlobIterator { + return &PackBlobIterator{ + packID: packID, + rd: rd, + currentOffset: currentOffset, + blobs: blobs, + key: key, + dec: dec, + } +} + +// Next returns the next blob, an error or ErrPackEOF if all blobs were read +func (b *PackBlobIterator) Next() (PackBlobValue, error) { + if len(b.blobs) == 0 { + return PackBlobValue{}, ErrPackEOF + } + + entry := b.blobs[0] + b.blobs = b.blobs[1:] + + skipBytes := int(entry.Offset - b.currentOffset) + if skipBytes < 0 { + return PackBlobValue{}, errors.Errorf("overlapping blobs in pack %v", b.packID) + } + + _, err := b.rd.Discard(skipBytes) + if err != nil { + return PackBlobValue{}, err + } + b.currentOffset = entry.Offset + + h := restic.BlobHandle{ID: entry.ID, Type: entry.Type} + debug.Log(" process blob %v, skipped %d, %v", h, skipBytes, entry) + + if uint(cap(b.buf)) < entry.Length { + b.buf = make([]byte, entry.Length) + } + b.buf = b.buf[:entry.Length] + + n, err := io.ReadFull(b.rd, b.buf) + if err != nil { + debug.Log(" read error %v", err) + return PackBlobValue{}, errors.Wrap(err, "ReadFull") + } + + if n != len(b.buf) { + return PackBlobValue{}, errors.Errorf("read blob %v from %v: not enough bytes read, want %v, got %v", + h, b.packID.Str(), len(b.buf), n) + } + b.currentOffset = entry.Offset + entry.Length + + if int(entry.Length) <= b.key.NonceSize() { + debug.Log("%v", b.blobs) + return PackBlobValue{}, errors.Errorf("invalid blob length %v", entry) + } + + // decryption errors are likely permanent, give the caller a chance to skip them + nonce, ciphertext := b.buf[:b.key.NonceSize()], b.buf[b.key.NonceSize():] + plaintext, err := b.key.Open(ciphertext[:0], nonce, ciphertext, nil) + if err == nil && entry.IsCompressed() { + // DecodeAll will allocate a slice if it is not large enough since it + // knows the decompressed size (because we're using EncodeAll) + b.decode, err = b.dec.DecodeAll(plaintext, b.decode[:0]) + plaintext = b.decode + if err != nil { + err = errors.Errorf("decompressing blob %v failed: %v", h, err) + } + } + if err == nil { + id := restic.Hash(plaintext) + if !id.Equal(entry.ID) { + debug.Log("read blob %v/%v from %v: wrong data returned, hash is %v", + h.Type, h.ID, b.packID.Str(), id) + err = errors.Errorf("read blob %v from %v: wrong data returned, hash is %v", + h, b.packID.Str(), id) + } + } + + return PackBlobValue{entry.BlobHandle, plaintext, err}, nil +} + var zeroChunkOnce sync.Once var zeroChunkID restic.ID From 22d0c3f8dcc3042219a9cad20d32f641d683f058 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 31 Dec 2023 10:58:26 +0100 Subject: [PATCH 039/158] check: Use PackBlobIterator instead of StreamPack To only stream the content of a pack file once, check used StreamPack with a custom pack load function. This combination was always brittle and complicates using StreamPack everywhere else. Now that StreamPack internally uses PackBlobIterator use that primitive instead, which is a much better fit for what the check command requires. --- internal/checker/checker.go | 81 +++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 3bc0fac87..e6a7a9035 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -10,6 +10,7 @@ import ( "sort" "sync" + "github.com/klauspost/compress/zstd" "github.com/minio/sha256-simd" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/s3" @@ -526,7 +527,7 @@ func (c *Checker) GetPacks() map[restic.ID]int64 { } // checkPack reads a pack and checks the integrity of all blobs. -func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []restic.Blob, size int64, bufRd *bufio.Reader) error { +func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []restic.Blob, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error { debug.Log("checking pack %v", id.String()) if len(blobs) == 0 { @@ -557,49 +558,44 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []r // calculate hash on-the-fly while reading the pack and capture pack header var hash restic.ID var hdrBuf []byte - hashingLoader := func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - return r.Backend().Load(ctx, h, int(size), 0, func(rd io.Reader) error { - hrd := hashing.NewReader(rd, sha256.New()) - bufRd.Reset(hrd) + h := backend.Handle{Type: backend.PackFile, Name: id.String()} + err := r.Backend().Load(ctx, h, int(size), 0, func(rd io.Reader) error { + hrd := hashing.NewReader(rd, sha256.New()) + bufRd.Reset(hrd) - // skip to start of first blob, offset == 0 for correct pack files - _, err := bufRd.Discard(int(offset)) - if err != nil { + it := repository.NewPackBlobIterator(id, bufRd, 0, blobs, r.Key(), dec) + for { + val, err := it.Next() + if err == repository.ErrPackEOF { + break + } else if err != nil { return err } - - err = fn(bufRd) - if err != nil { - return err + debug.Log(" check blob %v: %v", val.Handle.ID, val.Handle) + if val.Err != nil { + debug.Log(" error verifying blob %v: %v", val.Handle.ID, err) + errs = append(errs, errors.Errorf("blob %v: %v", val.Handle.ID, err)) } - - // skip enough bytes until we reach the possible header start - curPos := length + int(offset) - minHdrStart := int(size) - pack.MaxHeaderSize - if minHdrStart > curPos { - _, err := bufRd.Discard(minHdrStart - curPos) - if err != nil { - return err - } - } - - // read remainder, which should be the pack header - hdrBuf, err = io.ReadAll(bufRd) - if err != nil { - return err - } - - hash = restic.IDFromHash(hrd.Sum(nil)) - return nil - }) - } - - err := repository.StreamPack(ctx, hashingLoader, r.Key(), id, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { - debug.Log(" check blob %v: %v", blob.ID, blob) - if err != nil { - debug.Log(" error verifying blob %v: %v", blob.ID, err) - errs = append(errs, errors.Errorf("blob %v: %v", blob.ID, err)) } + + // skip enough bytes until we reach the possible header start + curPos := lastBlobEnd + minHdrStart := int(size) - pack.MaxHeaderSize + if minHdrStart > curPos { + _, err := bufRd.Discard(minHdrStart - curPos) + if err != nil { + return err + } + } + + // read remainder, which should be the pack header + var err error + hdrBuf, err = io.ReadAll(bufRd) + if err != nil { + return err + } + + hash = restic.IDFromHash(hrd.Sum(nil)) return nil }) if err != nil { @@ -670,6 +666,11 @@ func (c *Checker) ReadPacks(ctx context.Context, packs map[restic.ID]int64, p *p // create a buffer that is large enough to be reused by repository.StreamPack // this ensures that we can read the pack header later on bufRd := bufio.NewReaderSize(nil, repository.MaxStreamBufferSize) + dec, err := zstd.NewReader(nil) + if err != nil { + panic(dec) + } + defer dec.Close() for { var ps checkTask var ok bool @@ -683,7 +684,7 @@ func (c *Checker) ReadPacks(ctx context.Context, packs map[restic.ID]int64, p *p } } - err := checkPack(ctx, c.repo, ps.id, ps.blobs, ps.size, bufRd) + err := checkPack(ctx, c.repo, ps.id, ps.blobs, ps.size, bufRd, dec) p.Add(1) if err == nil { continue From 6b7b5c89e9109210fb7bc9933677d8a54d67866b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 31 Dec 2023 15:45:10 +0100 Subject: [PATCH 040/158] repository: prepare StreamPack refactor --- .../repository/repository_internal_test.go | 278 ++++++++++++++++++ internal/repository/repository_test.go | 273 ----------------- 2 files changed, 278 insertions(+), 273 deletions(-) diff --git a/internal/repository/repository_internal_test.go b/internal/repository/repository_internal_test.go index d8e35b993..fc408910c 100644 --- a/internal/repository/repository_internal_test.go +++ b/internal/repository/repository_internal_test.go @@ -1,11 +1,21 @@ package repository import ( + "bytes" + "context" + "encoding/json" + "io" "math/rand" "sort" + "strings" "testing" + "github.com/cenkalti/backoff/v4" + "github.com/google/go-cmp/cmp" + "github.com/klauspost/compress/zstd" "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/crypto" + "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -73,3 +83,271 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) { sortCachedPacksFirst(cache, cpy[:]) } } + +// buildPackfileWithoutHeader returns a manually built pack file without a header. +func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool) (blobs []restic.Blob, packfile []byte) { + opts := []zstd.EOption{ + // Set the compression level configured. + zstd.WithEncoderLevel(zstd.SpeedDefault), + // Disable CRC, we have enough checks in place, makes the + // compressed data four bytes shorter. + zstd.WithEncoderCRC(false), + // Set a window of 512kbyte, so we have good lookbehind for usual + // blob sizes. + zstd.WithWindowSize(512 * 1024), + } + enc, err := zstd.NewWriter(nil, opts...) + if err != nil { + panic(err) + } + + var offset uint + for i, size := range blobSizes { + plaintext := rtest.Random(800+i, size) + id := restic.Hash(plaintext) + uncompressedLength := uint(0) + if compress { + uncompressedLength = uint(len(plaintext)) + plaintext = enc.EncodeAll(plaintext, nil) + } + + // we use a deterministic nonce here so the whole process is + // deterministic, last byte is the blob index + var nonce = []byte{ + 0x15, 0x98, 0xc0, 0xf7, 0xb9, 0x65, 0x97, 0x74, + 0x12, 0xdc, 0xd3, 0x62, 0xa9, 0x6e, 0x20, byte(i), + } + + before := len(packfile) + packfile = append(packfile, nonce...) + packfile = key.Seal(packfile, nonce, plaintext, nil) + after := len(packfile) + + ciphertextLength := after - before + + blobs = append(blobs, restic.Blob{ + BlobHandle: restic.BlobHandle{ + Type: restic.DataBlob, + ID: id, + }, + Length: uint(ciphertextLength), + UncompressedLength: uncompressedLength, + Offset: offset, + }) + + offset = uint(len(packfile)) + } + + return blobs, packfile +} + +func TestStreamPack(t *testing.T) { + TestAllVersions(t, testStreamPack) +} + +func testStreamPack(t *testing.T, version uint) { + // always use the same key for deterministic output + const jsonKey = `{"mac":{"k":"eQenuI8adktfzZMuC8rwdA==","r":"k8cfAly2qQSky48CQK7SBA=="},"encrypt":"MKO9gZnRiQFl8mDUurSDa9NMjiu9MUifUrODTHS05wo="}` + + var key crypto.Key + err := json.Unmarshal([]byte(jsonKey), &key) + if err != nil { + t.Fatal(err) + } + + blobSizes := []int{ + 5522811, + 10, + 5231, + 18812, + 123123, + 13522811, + 12301, + 892242, + 28616, + 13351, + 252287, + 188883, + 3522811, + 18883, + } + + var compress bool + switch version { + case 1: + compress = false + case 2: + compress = true + default: + t.Fatal("test does not support repository version", version) + } + + packfileBlobs, packfile := buildPackfileWithoutHeader(blobSizes, &key, compress) + + loadCalls := 0 + shortFirstLoad := false + + loadBytes := func(length int, offset int64) []byte { + data := packfile + + if offset > int64(len(data)) { + offset = 0 + length = 0 + } + data = data[offset:] + + if length > len(data) { + length = len(data) + } + if shortFirstLoad { + length /= 2 + shortFirstLoad = false + } + + return data[:length] + } + + load := func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + data := loadBytes(length, offset) + if shortFirstLoad { + data = data[:len(data)/2] + shortFirstLoad = false + } + + loadCalls++ + + err := fn(bytes.NewReader(data)) + if err == nil { + return nil + } + var permanent *backoff.PermanentError + if errors.As(err, &permanent) { + return err + } + + // retry loading once + return fn(bytes.NewReader(loadBytes(length, offset))) + } + + // first, test regular usage + t.Run("regular", func(t *testing.T) { + tests := []struct { + blobs []restic.Blob + calls int + shortFirstLoad bool + }{ + {packfileBlobs[1:2], 1, false}, + {packfileBlobs[2:5], 1, false}, + {packfileBlobs[2:8], 1, false}, + {[]restic.Blob{ + packfileBlobs[0], + packfileBlobs[4], + packfileBlobs[2], + }, 1, false}, + {[]restic.Blob{ + packfileBlobs[0], + packfileBlobs[len(packfileBlobs)-1], + }, 2, false}, + {packfileBlobs[:], 1, true}, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + gotBlobs := make(map[restic.ID]int) + + handleBlob := func(blob restic.BlobHandle, buf []byte, err error) error { + gotBlobs[blob.ID]++ + + id := restic.Hash(buf) + if !id.Equal(blob.ID) { + t.Fatalf("wrong id %v for blob %s returned", id, blob.ID) + } + + return err + } + + wantBlobs := make(map[restic.ID]int) + for _, blob := range test.blobs { + wantBlobs[blob.ID] = 1 + } + + loadCalls = 0 + shortFirstLoad = test.shortFirstLoad + err = StreamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob) + if err != nil { + t.Fatal(err) + } + + if !cmp.Equal(wantBlobs, gotBlobs) { + t.Fatal(cmp.Diff(wantBlobs, gotBlobs)) + } + rtest.Equals(t, test.calls, loadCalls) + }) + } + }) + shortFirstLoad = false + + // next, test invalid uses, which should return an error + t.Run("invalid", func(t *testing.T) { + tests := []struct { + blobs []restic.Blob + err string + }{ + { + // pass one blob several times + blobs: []restic.Blob{ + packfileBlobs[3], + packfileBlobs[8], + packfileBlobs[3], + packfileBlobs[4], + }, + err: "overlapping blobs in pack", + }, + + { + // pass something that's not a valid blob in the current pack file + blobs: []restic.Blob{ + { + Offset: 123, + Length: 20000, + }, + }, + err: "ciphertext verification failed", + }, + + { + // pass a blob that's too small + blobs: []restic.Blob{ + { + Offset: 123, + Length: 10, + }, + }, + err: "invalid blob length", + }, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + handleBlob := func(blob restic.BlobHandle, buf []byte, err error) error { + return err + } + + err = StreamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob) + if err == nil { + t.Fatalf("wanted error %v, got nil", test.err) + } + + if !strings.Contains(err.Error(), test.err) { + t.Fatalf("wrong error returned, it should contain %q but was %q", test.err, err) + } + }) + } + }) +} diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 1178a7693..272ea94ac 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -4,8 +4,6 @@ import ( "bytes" "context" "crypto/sha256" - "encoding/json" - "errors" "fmt" "io" "math/rand" @@ -15,9 +13,6 @@ import ( "testing" "time" - "github.com/cenkalti/backoff/v4" - "github.com/google/go-cmp/cmp" - "github.com/klauspost/compress/zstd" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/local" "github.com/restic/restic/internal/crypto" @@ -430,274 +425,6 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) { } -// buildPackfileWithoutHeader returns a manually built pack file without a header. -func buildPackfileWithoutHeader(blobSizes []int, key *crypto.Key, compress bool) (blobs []restic.Blob, packfile []byte) { - opts := []zstd.EOption{ - // Set the compression level configured. - zstd.WithEncoderLevel(zstd.SpeedDefault), - // Disable CRC, we have enough checks in place, makes the - // compressed data four bytes shorter. - zstd.WithEncoderCRC(false), - // Set a window of 512kbyte, so we have good lookbehind for usual - // blob sizes. - zstd.WithWindowSize(512 * 1024), - } - enc, err := zstd.NewWriter(nil, opts...) - if err != nil { - panic(err) - } - - var offset uint - for i, size := range blobSizes { - plaintext := rtest.Random(800+i, size) - id := restic.Hash(plaintext) - uncompressedLength := uint(0) - if compress { - uncompressedLength = uint(len(plaintext)) - plaintext = enc.EncodeAll(plaintext, nil) - } - - // we use a deterministic nonce here so the whole process is - // deterministic, last byte is the blob index - var nonce = []byte{ - 0x15, 0x98, 0xc0, 0xf7, 0xb9, 0x65, 0x97, 0x74, - 0x12, 0xdc, 0xd3, 0x62, 0xa9, 0x6e, 0x20, byte(i), - } - - before := len(packfile) - packfile = append(packfile, nonce...) - packfile = key.Seal(packfile, nonce, plaintext, nil) - after := len(packfile) - - ciphertextLength := after - before - - blobs = append(blobs, restic.Blob{ - BlobHandle: restic.BlobHandle{ - Type: restic.DataBlob, - ID: id, - }, - Length: uint(ciphertextLength), - UncompressedLength: uncompressedLength, - Offset: offset, - }) - - offset = uint(len(packfile)) - } - - return blobs, packfile -} - -func TestStreamPack(t *testing.T) { - repository.TestAllVersions(t, testStreamPack) -} - -func testStreamPack(t *testing.T, version uint) { - // always use the same key for deterministic output - const jsonKey = `{"mac":{"k":"eQenuI8adktfzZMuC8rwdA==","r":"k8cfAly2qQSky48CQK7SBA=="},"encrypt":"MKO9gZnRiQFl8mDUurSDa9NMjiu9MUifUrODTHS05wo="}` - - var key crypto.Key - err := json.Unmarshal([]byte(jsonKey), &key) - if err != nil { - t.Fatal(err) - } - - blobSizes := []int{ - 5522811, - 10, - 5231, - 18812, - 123123, - 13522811, - 12301, - 892242, - 28616, - 13351, - 252287, - 188883, - 3522811, - 18883, - } - - var compress bool - switch version { - case 1: - compress = false - case 2: - compress = true - default: - t.Fatal("test does not support repository version", version) - } - - packfileBlobs, packfile := buildPackfileWithoutHeader(blobSizes, &key, compress) - - loadCalls := 0 - shortFirstLoad := false - - loadBytes := func(length int, offset int64) []byte { - data := packfile - - if offset > int64(len(data)) { - offset = 0 - length = 0 - } - data = data[offset:] - - if length > len(data) { - length = len(data) - } - if shortFirstLoad { - length /= 2 - shortFirstLoad = false - } - - return data[:length] - } - - load := func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - data := loadBytes(length, offset) - if shortFirstLoad { - data = data[:len(data)/2] - shortFirstLoad = false - } - - loadCalls++ - - err := fn(bytes.NewReader(data)) - if err == nil { - return nil - } - var permanent *backoff.PermanentError - if errors.As(err, &permanent) { - return err - } - - // retry loading once - return fn(bytes.NewReader(loadBytes(length, offset))) - } - - // first, test regular usage - t.Run("regular", func(t *testing.T) { - tests := []struct { - blobs []restic.Blob - calls int - shortFirstLoad bool - }{ - {packfileBlobs[1:2], 1, false}, - {packfileBlobs[2:5], 1, false}, - {packfileBlobs[2:8], 1, false}, - {[]restic.Blob{ - packfileBlobs[0], - packfileBlobs[4], - packfileBlobs[2], - }, 1, false}, - {[]restic.Blob{ - packfileBlobs[0], - packfileBlobs[len(packfileBlobs)-1], - }, 2, false}, - {packfileBlobs[:], 1, true}, - } - - for _, test := range tests { - t.Run("", func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - gotBlobs := make(map[restic.ID]int) - - handleBlob := func(blob restic.BlobHandle, buf []byte, err error) error { - gotBlobs[blob.ID]++ - - id := restic.Hash(buf) - if !id.Equal(blob.ID) { - t.Fatalf("wrong id %v for blob %s returned", id, blob.ID) - } - - return err - } - - wantBlobs := make(map[restic.ID]int) - for _, blob := range test.blobs { - wantBlobs[blob.ID] = 1 - } - - loadCalls = 0 - shortFirstLoad = test.shortFirstLoad - err = repository.StreamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob) - if err != nil { - t.Fatal(err) - } - - if !cmp.Equal(wantBlobs, gotBlobs) { - t.Fatal(cmp.Diff(wantBlobs, gotBlobs)) - } - rtest.Equals(t, test.calls, loadCalls) - }) - } - }) - shortFirstLoad = false - - // next, test invalid uses, which should return an error - t.Run("invalid", func(t *testing.T) { - tests := []struct { - blobs []restic.Blob - err string - }{ - { - // pass one blob several times - blobs: []restic.Blob{ - packfileBlobs[3], - packfileBlobs[8], - packfileBlobs[3], - packfileBlobs[4], - }, - err: "overlapping blobs in pack", - }, - - { - // pass something that's not a valid blob in the current pack file - blobs: []restic.Blob{ - { - Offset: 123, - Length: 20000, - }, - }, - err: "ciphertext verification failed", - }, - - { - // pass a blob that's too small - blobs: []restic.Blob{ - { - Offset: 123, - Length: 10, - }, - }, - err: "invalid blob length", - }, - } - - for _, test := range tests { - t.Run("", func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - handleBlob := func(blob restic.BlobHandle, buf []byte, err error) error { - return err - } - - err = repository.StreamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob) - if err == nil { - t.Fatalf("wanted error %v, got nil", test.err) - } - - if !strings.Contains(err.Error(), test.err) { - t.Fatalf("wrong error returned, it should contain %q but was %q", test.err, err) - } - }) - } - }) -} - func TestInvalidCompression(t *testing.T) { var comp repository.CompressionMode err := comp.Set("nope") From 2c310a526e9c0c0c7e26313c4ee06a94328e6395 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 31 Dec 2023 12:07:19 +0100 Subject: [PATCH 041/158] repository: Replace StreamPack function with LoadBlobsFromPack method LoadBlobsFromPack is now part of the repository struct. This ensures that users of that method don't have to deal will internals of the repository implementation. The filerestorer tests now also contain far fewer pack file implementation details. --- cmd/restic/cmd_repair_packs.go | 2 +- internal/repository/repack.go | 2 +- internal/repository/repository.go | 14 +- .../repository/repository_internal_test.go | 4 +- internal/restic/repository.go | 1 + internal/restorer/filerestorer.go | 16 +-- internal/restorer/filerestorer_test.go | 132 ++++++------------ internal/restorer/restorer.go | 2 +- 8 files changed, 66 insertions(+), 107 deletions(-) diff --git a/cmd/restic/cmd_repair_packs.go b/cmd/restic/cmd_repair_packs.go index 7d1a3a392..723bdbccb 100644 --- a/cmd/restic/cmd_repair_packs.go +++ b/cmd/restic/cmd_repair_packs.go @@ -116,7 +116,7 @@ func repairPacks(ctx context.Context, gopts GlobalOptions, repo *repository.Repo continue } - err = repository.StreamPack(wgCtx, repo.Backend().Load, repo.Key(), b.PackID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { + err = repo.LoadBlobsFromPack(wgCtx, b.PackID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { if err != nil { // Fallback path buf, err = repo.LoadBlob(wgCtx, blob.Type, blob.ID, nil) diff --git a/internal/repository/repack.go b/internal/repository/repack.go index c82e63f28..5588984f6 100644 --- a/internal/repository/repack.go +++ b/internal/repository/repack.go @@ -77,7 +77,7 @@ func repack(ctx context.Context, repo restic.Repository, dstRepo restic.Reposito worker := func() error { for t := range downloadQueue { - err := StreamPack(wgCtx, repo.Backend().Load, repo.Key(), t.PackID, t.Blobs, func(blob restic.BlobHandle, buf []byte, err error) error { + err := repo.LoadBlobsFromPack(wgCtx, t.PackID, t.Blobs, func(blob restic.BlobHandle, buf []byte, err error) error { if err != nil { var ierr error // check whether we can get a valid copy somewhere else diff --git a/internal/repository/repository.go b/internal/repository/repository.go index e13220741..407b6429c 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -875,16 +875,20 @@ func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte return newID, known, size, err } -type BackendLoadFn func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error +type backendLoadFn func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error // Skip sections with more than 4MB unused blobs const maxUnusedRange = 4 * 1024 * 1024 -// StreamPack loads the listed blobs from the specified pack file. The plaintext blob is passed to +// LoadBlobsFromPack loads the listed blobs from the specified pack file. The plaintext blob is passed to // the handleBlobFn callback or an error if decryption failed or the blob hash does not match. // handleBlobFn is called at most once for each blob. If the callback returns an error, -// then StreamPack will abort and not retry it. -func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { +// then LoadBlobsFromPack will abort and not retry it. +func (r *Repository) LoadBlobsFromPack(ctx context.Context, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { + return streamPack(ctx, r.Backend().Load, r.key, packID, blobs, handleBlobFn) +} + +func streamPack(ctx context.Context, beLoad backendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { if len(blobs) == 0 { // nothing to do return nil @@ -915,7 +919,7 @@ func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, pack return streamPackPart(ctx, beLoad, key, packID, blobs[lowerIdx:], handleBlobFn) } -func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { +func streamPackPart(ctx context.Context, beLoad backendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { h := backend.Handle{Type: restic.PackFile, Name: packID.String(), IsMetadata: false} dataStart := blobs[0].Offset diff --git a/internal/repository/repository_internal_test.go b/internal/repository/repository_internal_test.go index fc408910c..eed99c7e0 100644 --- a/internal/repository/repository_internal_test.go +++ b/internal/repository/repository_internal_test.go @@ -276,7 +276,7 @@ func testStreamPack(t *testing.T, version uint) { loadCalls = 0 shortFirstLoad = test.shortFirstLoad - err = StreamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob) + err = streamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob) if err != nil { t.Fatal(err) } @@ -339,7 +339,7 @@ func testStreamPack(t *testing.T, version uint) { return err } - err = StreamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob) + err = streamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob) if err == nil { t.Fatalf("wanted error %v, got nil", test.err) } diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 895c930dd..6818847c0 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -44,6 +44,7 @@ type Repository interface { ListPack(context.Context, ID, int64) ([]Blob, uint32, error) LoadBlob(context.Context, BlobType, ID, []byte) ([]byte, error) + LoadBlobsFromPack(ctx context.Context, packID ID, blobs []Blob, handleBlobFn func(blob BlobHandle, buf []byte, err error) error) error SaveBlob(context.Context, BlobType, []byte, ID, bool) (ID, bool, int, error) // StartPackUploader start goroutines to upload new pack files. The errgroup diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index 99a460321..f2c134ea9 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -7,7 +7,6 @@ import ( "golang.org/x/sync/errgroup" - "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" @@ -45,11 +44,12 @@ type packInfo struct { files map[*fileInfo]struct{} // set of files that use blobs from this pack } +type blobsLoaderFn func(ctx context.Context, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error + // fileRestorer restores set of files type fileRestorer struct { - key *crypto.Key - idx func(restic.BlobHandle) []restic.PackedBlob - packLoader repository.BackendLoadFn + idx func(restic.BlobHandle) []restic.PackedBlob + blobsLoader blobsLoaderFn workerCount int filesWriter *filesWriter @@ -63,8 +63,7 @@ type fileRestorer struct { } func newFileRestorer(dst string, - packLoader repository.BackendLoadFn, - key *crypto.Key, + blobsLoader blobsLoaderFn, idx func(restic.BlobHandle) []restic.PackedBlob, connections uint, sparse bool, @@ -74,9 +73,8 @@ func newFileRestorer(dst string, workerCount := int(connections) return &fileRestorer{ - key: key, idx: idx, - packLoader: packLoader, + blobsLoader: blobsLoader, filesWriter: newFilesWriter(workerCount), zeroChunk: repository.ZeroChunk(), sparse: sparse, @@ -310,7 +308,7 @@ func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID, for _, entry := range blobs { blobList = append(blobList, entry.blob) } - return repository.StreamPack(ctx, r.packLoader, r.key, packID, blobList, + return r.blobsLoader(ctx, packID, blobList, func(h restic.BlobHandle, blobData []byte, err error) error { processedBlobs.Insert(h) blob := blobs[h.ID] diff --git a/internal/restorer/filerestorer_test.go b/internal/restorer/filerestorer_test.go index c5bc3fe31..befeb5d2c 100644 --- a/internal/restorer/filerestorer_test.go +++ b/internal/restorer/filerestorer_test.go @@ -4,14 +4,11 @@ import ( "bytes" "context" "fmt" - "io" "os" + "sort" "testing" - "github.com/restic/restic/internal/backend" - "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -27,11 +24,6 @@ type TestFile struct { } type TestRepo struct { - key *crypto.Key - - // pack names and ids - packsNameToID map[string]restic.ID - packsIDToName map[restic.ID]string packsIDToData map[restic.ID][]byte // blobs and files @@ -40,7 +32,7 @@ type TestRepo struct { filesPathToContent map[string]string // - loader repository.BackendLoadFn + loader blobsLoaderFn } func (i *TestRepo) Lookup(bh restic.BlobHandle) []restic.PackedBlob { @@ -59,16 +51,6 @@ func newTestRepo(content []TestFile) *TestRepo { blobs map[restic.ID]restic.Blob } packs := make(map[string]Pack) - - key := crypto.NewRandomKey() - seal := func(data []byte) []byte { - ciphertext := crypto.NewBlobBuffer(len(data)) - ciphertext = ciphertext[:0] // truncate the slice - nonce := crypto.NewRandomNonce() - ciphertext = append(ciphertext, nonce...) - return key.Seal(ciphertext, nonce, data, nil) - } - filesPathToContent := make(map[string]string) for _, file := range content { @@ -86,14 +68,15 @@ func newTestRepo(content []TestFile) *TestRepo { // calculate blob id and add to the pack as necessary blobID := restic.Hash([]byte(blob.data)) if _, found := pack.blobs[blobID]; !found { - blobData := seal([]byte(blob.data)) + blobData := []byte(blob.data) pack.blobs[blobID] = restic.Blob{ BlobHandle: restic.BlobHandle{ Type: restic.DataBlob, ID: blobID, }, - Length: uint(len(blobData)), - Offset: uint(len(pack.data)), + Length: uint(len(blobData)), + UncompressedLength: uint(len(blobData)), + Offset: uint(len(pack.data)), } pack.data = append(pack.data, blobData...) } @@ -104,15 +87,11 @@ func newTestRepo(content []TestFile) *TestRepo { } blobs := make(map[restic.ID][]restic.PackedBlob) - packsIDToName := make(map[restic.ID]string) packsIDToData := make(map[restic.ID][]byte) - packsNameToID := make(map[string]restic.ID) for _, pack := range packs { packID := restic.Hash(pack.data) - packsIDToName[packID] = pack.name packsIDToData[packID] = pack.data - packsNameToID[pack.name] = packID for blobID, blob := range pack.blobs { blobs[blobID] = append(blobs[blobID], restic.PackedBlob{Blob: blob, PackID: packID}) } @@ -128,30 +107,44 @@ func newTestRepo(content []TestFile) *TestRepo { } repo := &TestRepo{ - key: key, - packsIDToName: packsIDToName, packsIDToData: packsIDToData, - packsNameToID: packsNameToID, blobs: blobs, files: files, filesPathToContent: filesPathToContent, } - repo.loader = func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - packID, err := restic.ParseID(h.Name) - if err != nil { - return err + repo.loader = func(ctx context.Context, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { + blobs = append([]restic.Blob{}, blobs...) + sort.Slice(blobs, func(i, j int) bool { + return blobs[i].Offset < blobs[j].Offset + }) + + for _, blob := range blobs { + found := false + for _, e := range repo.blobs[blob.ID] { + if packID == e.PackID { + found = true + buf := repo.packsIDToData[packID][e.Offset : e.Offset+e.Length] + err := handleBlobFn(e.BlobHandle, buf, nil) + if err != nil { + return err + } + } + } + if !found { + return fmt.Errorf("missing blob: %v", blob) + } } - rd := bytes.NewReader(repo.packsIDToData[packID][int(offset) : int(offset)+length]) - return fn(rd) + return nil } return repo } func restoreAndVerify(t *testing.T, tempdir string, content []TestFile, files map[string]bool, sparse bool) { + t.Helper() repo := newTestRepo(content) - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, sparse, nil) + r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, sparse, nil) if files == nil { r.files = repo.files @@ -170,6 +163,7 @@ func restoreAndVerify(t *testing.T, tempdir string, content []TestFile, files ma } func verifyRestore(t *testing.T, r *fileRestorer, repo *TestRepo) { + t.Helper() for _, file := range r.files { target := r.targetPath(file.location) data, err := os.ReadFile(target) @@ -283,62 +277,17 @@ func TestErrorRestoreFiles(t *testing.T) { loadError := errors.New("load error") // loader always returns an error - repo.loader = func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + repo.loader = func(ctx context.Context, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { return loadError } - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false, nil) + r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, false, nil) r.files = repo.files err := r.restoreFiles(context.TODO()) rtest.Assert(t, errors.Is(err, loadError), "got %v, expected contained error %v", err, loadError) } -func TestDownloadError(t *testing.T) { - for i := 0; i < 100; i += 10 { - testPartialDownloadError(t, i) - } -} - -func testPartialDownloadError(t *testing.T, part int) { - tempdir := rtest.TempDir(t) - content := []TestFile{ - { - name: "file1", - blobs: []TestBlob{ - {"data1-1", "pack1"}, - {"data1-2", "pack1"}, - {"data1-3", "pack1"}, - }, - }} - - repo := newTestRepo(content) - - // loader always returns an error - loader := repo.loader - repo.loader = func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - // only load partial data to exercise fault handling in different places - err := loader(ctx, h, length*part/100, offset, fn) - if err == nil { - return nil - } - fmt.Println("Retry after error", err) - return loader(ctx, h, length, offset, fn) - } - - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false, nil) - r.files = repo.files - r.Error = func(s string, e error) error { - // ignore errors as in the `restore` command - fmt.Println("error during restore", s, e) - return nil - } - - err := r.restoreFiles(context.TODO()) - rtest.OK(t, err) - verifyRestore(t, r, repo) -} - func TestFatalDownloadError(t *testing.T) { tempdir := rtest.TempDir(t) content := []TestFile{ @@ -361,12 +310,19 @@ func TestFatalDownloadError(t *testing.T) { repo := newTestRepo(content) loader := repo.loader - repo.loader = func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { - // only return half the data to break file2 - return loader(ctx, h, length/2, offset, fn) + repo.loader = func(ctx context.Context, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { + ctr := 0 + return loader(ctx, packID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { + if ctr < 2 { + ctr++ + return handleBlobFn(blob, buf, err) + } + // break file2 + return errors.New("failed to load blob") + }) } - r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false, nil) + r := newFileRestorer(tempdir, repo.loader, repo.Lookup, 2, false, nil) r.files = repo.files var errors []string diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index e973316c0..2ce1ee98e 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -231,7 +231,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { } idx := NewHardlinkIndex[string]() - filerestorer := newFileRestorer(dst, res.repo.Backend().Load, res.repo.Key(), res.repo.Index().Lookup, + filerestorer := newFileRestorer(dst, res.repo.LoadBlobsFromPack, res.repo.Index().Lookup, res.repo.Connections(), res.sparse, res.progress) filerestorer.Error = res.Error From 8ca58b487cd7c6f4df04b2dd874b9eb6becf4e4e Mon Sep 17 00:00:00 2001 From: Vladislav Belous Date: Tue, 5 Dec 2023 00:20:27 +0200 Subject: [PATCH 042/158] S3: do not set storage class for metadata when using archive storage --- changelog/unreleased/issue-4583 | 9 +++++++++ internal/backend/s3/s3.go | 14 ++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/issue-4583 diff --git a/changelog/unreleased/issue-4583 b/changelog/unreleased/issue-4583 new file mode 100644 index 000000000..6af8ab475 --- /dev/null +++ b/changelog/unreleased/issue-4583 @@ -0,0 +1,9 @@ +Bugfix: Ignoring the s3.storage-class option for metadata when archive tier is specified + +Restic now will save snapshot metadata to non-archive storage tier whatsoever, +this will help avoid issues when data is being saved to archive storage class. +It is not providing any support for cold storages in restic, +only saving users from making backups unusable. + +https://github.com/restic/restic/issues/4583 +https://github.com/restic/restic/issues/3202 \ No newline at end of file diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index f0447224f..f9947b3a4 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -325,12 +325,22 @@ func (be *Backend) Path() string { return be.cfg.Prefix } +// useStorageClass returns whether file should be saved in the provided Storage Class +func (be *Backend) useStorageClass(h backend.Handle) bool { + var notArchiveClass bool = be.cfg.StorageClass != "GLACIER" && be.cfg.StorageClass != "DEEP_ARCHIVE" + isDataFile := h.Type == backend.PackFile && !h.IsMetadata + return isDataFile || notArchiveClass +} + // Save stores data in the backend at the handle. func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { objName := be.Filename(h) - opts := minio.PutObjectOptions{StorageClass: be.cfg.StorageClass} - opts.ContentType = "application/octet-stream" + opts := minio.PutObjectOptions{ContentType: "application/octet-stream"} + + if be.useStorageClass(h) { + opts.StorageClass = be.cfg.StorageClass + } // the only option with the high-level api is to let the library handle the checksum computation opts.SendContentMd5 = true // only use multipart uploads for very large files From a763a5c67de69b45eb1d56acaf4f639bae9fb891 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 11:18:09 +0100 Subject: [PATCH 043/158] s3: minor cleanups for archive storage class handling --- changelog/unreleased/issue-4583 | 15 +++++++++------ internal/backend/s3/s3.go | 17 ++++++++++------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/changelog/unreleased/issue-4583 b/changelog/unreleased/issue-4583 index 6af8ab475..97b0e6ba7 100644 --- a/changelog/unreleased/issue-4583 +++ b/changelog/unreleased/issue-4583 @@ -1,9 +1,12 @@ -Bugfix: Ignoring the s3.storage-class option for metadata when archive tier is specified +Enhancement: Ignore s3.storage-class for metadata if archive tier is specified -Restic now will save snapshot metadata to non-archive storage tier whatsoever, -this will help avoid issues when data is being saved to archive storage class. -It is not providing any support for cold storages in restic, -only saving users from making backups unusable. +There is no official cold storage support in restic, use this option at your +own risk. + +Restic always stored all files on s3 using the specified `s3.storage-class`. +Now, restic will store metadata using a non-archive storage tier to avoid +problems when accessing a repository. To restore any data, it is still +necessary to manually warm up the required data beforehand. https://github.com/restic/restic/issues/4583 -https://github.com/restic/restic/issues/3202 \ No newline at end of file +https://github.com/restic/restic/pull/4584 diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index f9947b3a4..d41f4479d 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -326,8 +326,10 @@ func (be *Backend) Path() string { } // useStorageClass returns whether file should be saved in the provided Storage Class +// For archive storage classes, only data files are stored using that class; metadata +// must remain instantly accessible. func (be *Backend) useStorageClass(h backend.Handle) bool { - var notArchiveClass bool = be.cfg.StorageClass != "GLACIER" && be.cfg.StorageClass != "DEEP_ARCHIVE" + notArchiveClass := be.cfg.StorageClass != "GLACIER" && be.cfg.StorageClass != "DEEP_ARCHIVE" isDataFile := h.Type == backend.PackFile && !h.IsMetadata return isDataFile || notArchiveClass } @@ -336,15 +338,16 @@ func (be *Backend) useStorageClass(h backend.Handle) bool { func (be *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) error { objName := be.Filename(h) - opts := minio.PutObjectOptions{ContentType: "application/octet-stream"} - + opts := minio.PutObjectOptions{ + ContentType: "application/octet-stream", + // the only option with the high-level api is to let the library handle the checksum computation + SendContentMd5: true, + // only use multipart uploads for very large files + PartSize: 200 * 1024 * 1024, + } if be.useStorageClass(h) { opts.StorageClass = be.cfg.StorageClass } - // the only option with the high-level api is to let the library handle the checksum computation - opts.SendContentMd5 = true - // only use multipart uploads for very large files - opts.PartSize = 200 * 1024 * 1024 info, err := be.client.PutObject(ctx, be.cfg.Bucket, objName, io.NopCloser(rd), int64(rd.Length()), opts) From fee83e1c09e3fa5b4cc2c71c00b275f035efb118 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 12:36:59 +0100 Subject: [PATCH 044/158] Remove redundant poly1305 key masking The implementation in crypto/poly1305 already performs the exact same masking. --- internal/crypto/crypto.go | 39 --------------------------------------- 1 file changed, 39 deletions(-) diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go index 752d886e3..0f9179207 100644 --- a/internal/crypto/crypto.go +++ b/internal/crypto/crypto.go @@ -45,28 +45,6 @@ type EncryptionKey [32]byte type MACKey struct { K [16]byte // for AES-128 R [16]byte // for Poly1305 - - masked bool // remember if the MAC key has already been masked -} - -// mask for key, (cf. http://cr.yp.to/mac/poly1305-20050329.pdf) -var poly1305KeyMask = [16]byte{ - 0xff, - 0xff, - 0xff, - 0x0f, // 3: top four bits zero - 0xfc, // 4: bottom two bits zero - 0xff, - 0xff, - 0x0f, // 7: top four bits zero - 0xfc, // 8: bottom two bits zero - 0xff, - 0xff, - 0x0f, // 11: top four bits zero - 0xfc, // 12: bottom two bits zero - 0xff, - 0xff, - 0x0f, // 15: top four bits zero } func poly1305MAC(msg []byte, nonce []byte, key *MACKey) []byte { @@ -78,32 +56,16 @@ func poly1305MAC(msg []byte, nonce []byte, key *MACKey) []byte { return out[:] } -// mask poly1305 key -func maskKey(k *MACKey) { - if k == nil || k.masked { - return - } - - for i := 0; i < poly1305.TagSize; i++ { - k.R[i] = k.R[i] & poly1305KeyMask[i] - } - - k.masked = true -} - // construct mac key from slice (k||r), with masking func macKeyFromSlice(mk *MACKey, data []byte) { copy(mk.K[:], data[:16]) copy(mk.R[:], data[16:32]) - maskKey(mk) } // prepare key for low-level poly1305.Sum(): r||n func poly1305PrepareKey(nonce []byte, key *MACKey) [32]byte { var k [32]byte - maskKey(key) - cipher, err := aes.NewCipher(key.K[:]) if err != nil { panic(err) @@ -143,7 +105,6 @@ func NewRandomKey() *Key { panic("unable to read enough random bytes for MAC key") } - maskKey(&k.MACKey) return k } From f1f34eb3e5830ec9cfecfe169c751dede0fe5ca1 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 22:28:10 +0100 Subject: [PATCH 045/158] lock: checkForOtherLocks processes each lock at most once If a lock could not be loaded, then restic would check all lock files again. These repeated checks are not useful as the status of a lock file cannot change unless its ID changes too. Thus, skip already check lock files on retries. --- internal/restic/lock.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/internal/restic/lock.go b/internal/restic/lock.go index 175cf6188..2d7e04f87 100644 --- a/internal/restic/lock.go +++ b/internal/restic/lock.go @@ -163,9 +163,16 @@ func (l *Lock) fillUserInfo() error { // exclusive lock is found. func (l *Lock) checkForOtherLocks(ctx context.Context) error { var err error + checkedIDs := NewIDSet() + if l.lockID != nil { + checkedIDs.Insert(*l.lockID) + } // retry locking a few times for i := 0; i < 3; i++ { - err = ForAllLocks(ctx, l.repo, l.lockID, func(id ID, lock *Lock, err error) error { + // Store updates in new IDSet to prevent data races + var m sync.Mutex + newCheckedIDs := NewIDSet(checkedIDs.List()...) + err = ForAllLocks(ctx, l.repo, checkedIDs, func(id ID, lock *Lock, err error) error { if err != nil { // if we cannot load a lock then it is unclear whether it can be ignored // it could either be invalid or just unreadable due to network/permission problems @@ -181,8 +188,13 @@ func (l *Lock) checkForOtherLocks(ctx context.Context) error { return &alreadyLockedError{otherLock: lock} } + // valid locks will remain valid + m.Lock() + newCheckedIDs.Insert(id) + m.Unlock() return nil }) + checkedIDs = newCheckedIDs // no lock detected if err == nil { return nil @@ -417,12 +429,12 @@ func RemoveAllLocks(ctx context.Context, repo Repository) (uint, error) { // It is guaranteed that the function is not run concurrently. If the // callback returns an error, this function is cancelled and also returns that error. // If a lock ID is passed via excludeID, it will be ignored. -func ForAllLocks(ctx context.Context, repo Repository, excludeID *ID, fn func(ID, *Lock, error) error) error { +func ForAllLocks(ctx context.Context, repo Repository, excludeIDs IDSet, fn func(ID, *Lock, error) error) error { var m sync.Mutex // For locks decoding is nearly for free, thus just assume were only limited by IO return ParallelList(ctx, repo, LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error { - if excludeID != nil && id.Equal(*excludeID) { + if excludeIDs.Has(id) { return nil } if size == 0 { From 6bdca136033df830dbefcb82b30b841cf613e6f4 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 21 Jan 2024 21:16:33 +0100 Subject: [PATCH 046/158] ui/termstatus: simplify cleaning up on termination `writeStatus` also cleans no longer used status lines. The old code actually cleaned one line too much. However, as that line was never used it makes no difference. --- internal/ui/termstatus/status.go | 26 +------------------------- internal/ui/termstatus/status_test.go | 5 ++--- 2 files changed, 3 insertions(+), 28 deletions(-) diff --git a/internal/ui/termstatus/status.go b/internal/ui/termstatus/status.go index 95286de99..fc731b023 100644 --- a/internal/ui/termstatus/status.go +++ b/internal/ui/termstatus/status.go @@ -105,7 +105,7 @@ func (t *Terminal) run(ctx context.Context) { select { case <-ctx.Done(): if !IsProcessBackground(t.fd) { - t.undoStatus(len(status)) + t.writeStatus([]string{}) } return @@ -235,30 +235,6 @@ func (t *Terminal) runWithoutStatus(ctx context.Context) { } } -func (t *Terminal) undoStatus(lines int) { - for i := 0; i < lines; i++ { - t.clearCurrentLine(t.wr, t.fd) - - _, err := t.wr.WriteRune('\n') - if err != nil { - fmt.Fprintf(os.Stderr, "write failed: %v\n", err) - } - - // flush is needed so that the current line is updated - err = t.wr.Flush() - if err != nil { - fmt.Fprintf(os.Stderr, "flush failed: %v\n", err) - } - } - - t.moveCursorUp(t.wr, t.fd, lines) - - err := t.wr.Flush() - if err != nil { - fmt.Fprintf(os.Stderr, "flush failed: %v\n", err) - } -} - func (t *Terminal) print(line string, isErr bool) { // make sure the line ends with a line break if line[len(line)-1] != '\n' { diff --git a/internal/ui/termstatus/status_test.go b/internal/ui/termstatus/status_test.go index b59063076..997a2d7b1 100644 --- a/internal/ui/termstatus/status_test.go +++ b/internal/ui/termstatus/status_test.go @@ -39,11 +39,10 @@ func TestSetStatus(t *testing.T) { term.SetStatus([]string{"quux", "needs\nquote"}) exp += home + clear + "quux\n" + home + clear + "\"needs\\nquote\"\n" + - home + clear + home + up + up // Third line implicit. + home + clear + home + up + up // Clear third line cancel() - exp += home + clear + "\n" + home + clear + "\n" + - home + up + up // Status cleared. + exp += home + clear + "\n" + home + clear + home + up // Status cleared <-term.closed rtest.Equals(t, exp, buf.String()) From 767c2539a0d573b9b4d7699ba9f28f9e28fb69ff Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 21 Jan 2024 22:06:54 +0100 Subject: [PATCH 047/158] backup: Improve help text for `--stdin-from-command` --- changelog/unreleased/issue-4251 | 9 +++++---- cmd/restic/cmd_backup.go | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/changelog/unreleased/issue-4251 b/changelog/unreleased/issue-4251 index 31be52401..5aeb50062 100644 --- a/changelog/unreleased/issue-4251 +++ b/changelog/unreleased/issue-4251 @@ -3,10 +3,11 @@ Enhancement: Support reading backup from a program's standard output When reading data from stdin, the `backup` command could not verify whether the corresponding command completed successfully. -The `backup` command now supports starting an arbitrary command and sourcing -the backup content from its standard output. This enables restic to verify that -the command completes with exit code zero. A non-zero exit code causes the -backup to fail. +The `backup` command now supports the `--stdin-from-command` option. When using +this option, the arguments to `backup` are interpreted as a command. `backup` +then executes the command and stores its standard output in the backup. This +enables restic to verify that the command completes with exit code zero. A +non-zero exit code causes the backup to fail. Example: `restic backup --stdin-from-command mysqldump [...]` diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index a2b81a759..be3de5f8e 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -135,7 +135,7 @@ func init() { f.StringVar(&backupOptions.ExcludeLargerThan, "exclude-larger-than", "", "max `size` of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)") f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin") f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin") - f.BoolVar(&backupOptions.StdinCommand, "stdin-from-command", false, "execute command and store its stdout") + f.BoolVar(&backupOptions.StdinCommand, "stdin-from-command", false, "interpret arguments as command to execute and store its stdout") f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)") f.UintVar(&backupOptions.ReadConcurrency, "read-concurrency", 0, "read `n` files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)") f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag") From 609f84e0953fa871029648a4e57545324689df48 Mon Sep 17 00:00:00 2001 From: adrian5 Date: Mon, 22 Jan 2024 21:12:12 +0100 Subject: [PATCH 048/158] docs: fix formatting --- doc/040_backup.rst | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/doc/040_backup.rst b/doc/040_backup.rst index d36986441..2e68527e2 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -173,7 +173,7 @@ attributes have to match for its contents to be presumed unchanged: * Modification timestamp (mtime). * Metadata change timestamp (ctime). * File size. - * Inode number (internal number used to reference a file in a filesystem). + * inode number (internal number used to reference a file in a filesystem). The reason for requiring both mtime and ctime to match is that Unix programs can freely change mtime (and some do). In such cases, a ctime change may be @@ -482,13 +482,12 @@ want to save the access time for files and directories, you can pass the ``--with-atime`` option to the ``backup`` command. Note that ``restic`` does not back up some metadata associated with files. Of -particular note are:: - - - file creation date on Unix platforms - - inode flags on Unix platforms - - file ownership and ACLs on Windows - - the "hidden" flag on Windows +particular note are: + * File creation date on Unix platforms + * inode flags on Unix platforms + * File ownership and ACLs on Windows + * The "hidden" flag on Windows Reading data from a command *************************** @@ -514,7 +513,6 @@ Restic uses the command exit code to determine whether the command succeeded. A non-zero exit code from the command causes restic to cancel the backup. This causes restic to fail with exit code 1. No snapshot will be created in this case. - Reading data from stdin *********************** @@ -555,7 +553,6 @@ the pipe and act accordingly (e.g., remove the last backup). Refer to the `Use the Unofficial Bash Strict Mode `__ for more details on this. - Tags for backup *************** @@ -684,7 +681,6 @@ The external programs that restic may execute include ``rclone`` (for rclone backends) and ``ssh`` (for the SFTP backend). These may respond to further environment variables and configuration files; see their respective manuals. - Exit status codes ***************** From 18806944f63b341756b98cca8aa2eab3b9952c45 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Tue, 23 Jan 2024 19:01:22 +0100 Subject: [PATCH 049/158] doc: remove blockquotes from unordered lists --- doc/030_preparing_a_new_repo.rst | 20 ++++++------- doc/040_backup.rst | 50 ++++++++++++++++---------------- doc/REST_backend.rst | 16 +++++----- doc/design.rst | 2 +- doc/developer_information.rst | 22 +++++++------- doc/manual_rest.rst | 8 ++--- 6 files changed, 59 insertions(+), 59 deletions(-) diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index 40215ad81..04fa45468 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -35,15 +35,15 @@ environment variable ``RESTIC_REPOSITORY_FILE``. For automating the supply of the repository password to restic, several options exist: - * Setting the environment variable ``RESTIC_PASSWORD`` +* Setting the environment variable ``RESTIC_PASSWORD`` - * Specifying the path to a file with the password via the option - ``--password-file`` or the environment variable ``RESTIC_PASSWORD_FILE`` +* Specifying the path to a file with the password via the option + ``--password-file`` or the environment variable ``RESTIC_PASSWORD_FILE`` + +* Configuring a program to be called when the password is needed via the + option ``--password-command`` or the environment variable + ``RESTIC_PASSWORD_COMMAND`` - * Configuring a program to be called when the password is needed via the - option ``--password-command`` or the environment variable - ``RESTIC_PASSWORD_COMMAND`` - The ``init`` command has an option called ``--repository-version`` which can be used to explicitly set the version of the new repository. By default, the current stable version is used (see table below). The alias ``latest`` will @@ -709,9 +709,9 @@ For debugging rclone, you can set the environment variable ``RCLONE_VERBOSE=2``. The rclone backend has three additional options: - * ``-o rclone.program`` specifies the path to rclone, the default value is just ``rclone`` - * ``-o rclone.args`` allows setting the arguments passed to rclone, by default this is ``serve restic --stdio --b2-hard-delete`` - * ``-o rclone.timeout`` specifies timeout for waiting on repository opening, the default value is ``1m`` +* ``-o rclone.program`` specifies the path to rclone, the default value is just ``rclone`` +* ``-o rclone.args`` allows setting the arguments passed to rclone, by default this is ``serve restic --stdio --b2-hard-delete`` +* ``-o rclone.timeout`` specifies timeout for waiting on repository opening, the default value is ``1m`` The reason for the ``--b2-hard-delete`` parameters can be found in the corresponding GitHub `issue #1657`_. diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 2e68527e2..ae834491e 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -170,10 +170,10 @@ On **Unix** (including Linux and Mac), given that a file lives at the same location as a file in a previous backup, the following file metadata attributes have to match for its contents to be presumed unchanged: - * Modification timestamp (mtime). - * Metadata change timestamp (ctime). - * File size. - * inode number (internal number used to reference a file in a filesystem). +* Modification timestamp (mtime). +* Metadata change timestamp (ctime). +* File size. +* Inode number (internal number used to reference a file in a filesystem). The reason for requiring both mtime and ctime to match is that Unix programs can freely change mtime (and some do). In such cases, a ctime change may be @@ -182,9 +182,9 @@ the only hint that a file did change. The following ``restic backup`` command line flags modify the change detection rules: - * ``--force``: turn off change detection and rescan all files. - * ``--ignore-ctime``: require mtime to match, but allow ctime to differ. - * ``--ignore-inode``: require mtime to match, but allow inode number +* ``--force``: turn off change detection and rescan all files. +* ``--ignore-ctime``: require mtime to match, but allow ctime to differ. +* ``--ignore-inode``: require mtime to match, but allow inode number and ctime to differ. The option ``--ignore-inode`` exists to support FUSE-based filesystems and @@ -250,9 +250,9 @@ It can be used like this: This instructs restic to exclude files matching the following criteria: - * All files matching ``*.c`` (parameter ``--exclude``) - * All files matching ``*.go`` (second line in ``excludes.txt``) - * All files and sub-directories named ``bar`` which reside somewhere below a directory called ``foo`` (fourth line in ``excludes.txt``) +* All files matching ``*.c`` (parameter ``--exclude``) +* All files matching ``*.go`` (second line in ``excludes.txt``) +* All files and sub-directories named ``bar`` which reside somewhere below a directory called ``foo`` (fourth line in ``excludes.txt``) Patterns use the syntax of the Go function `filepath.Match `__ @@ -270,8 +270,8 @@ environment variable (depending on your operating system). Patterns need to match on complete path components. For example, the pattern ``foo``: - * matches ``/dir1/foo/dir2/file`` and ``/dir/foo`` - * does not match ``/dir/foobar`` or ``barfoo`` +* matches ``/dir1/foo/dir2/file`` and ``/dir/foo`` +* does not match ``/dir/foobar`` or ``barfoo`` A trailing ``/`` is ignored, a leading ``/`` anchors the pattern at the root directory. This means, ``/bin`` matches ``/bin/bash`` but does not match ``/usr/bin/restic``. @@ -281,9 +281,9 @@ e.g. ``b*ash`` matches ``/bin/bash`` but does not match ``/bin/ash``. For this, the special wildcard ``**`` can be used to match arbitrary sub-directories: The pattern ``foo/**/bar`` matches: - * ``/dir1/foo/dir2/bar/file`` - * ``/foo/bar/file`` - * ``/tmp/foo/bar`` +* ``/dir1/foo/dir2/bar/file`` +* ``/foo/bar/file`` +* ``/tmp/foo/bar`` Spaces in patterns listed in an exclude file can be specified verbatim. That is, in order to exclude a file named ``foo bar star.txt``, put that just as it reads @@ -298,9 +298,9 @@ some escaping in order to pass the name/pattern as a single argument to restic. On most Unixy shells, you can either quote or use backslashes. For example: - * ``--exclude='foo bar star/foo.txt'`` - * ``--exclude="foo bar star/foo.txt"`` - * ``--exclude=foo\ bar\ star/foo.txt`` +* ``--exclude='foo bar star/foo.txt'`` +* ``--exclude="foo bar star/foo.txt"`` +* ``--exclude=foo\ bar\ star/foo.txt`` If a pattern starts with exclamation mark and matches a file that was previously matched by a regular pattern, the match is cancelled. @@ -484,10 +484,10 @@ want to save the access time for files and directories, you can pass the Note that ``restic`` does not back up some metadata associated with files. Of particular note are: - * File creation date on Unix platforms - * inode flags on Unix platforms - * File ownership and ACLs on Windows - * The "hidden" flag on Windows +* File creation date on Unix platforms +* Inode flags on Unix platforms +* File ownership and ACLs on Windows +* The "hidden" flag on Windows Reading data from a command *************************** @@ -686,9 +686,9 @@ Exit status codes Restic returns one of the following exit status codes after the backup command is run: - * 0 when the backup was successful (snapshot with all source files created) - * 1 when there was a fatal error (no snapshot created) - * 3 when some source files could not be read (incomplete snapshot with remaining files created) +* 0 when the backup was successful (snapshot with all source files created) +* 1 when there was a fatal error (no snapshot created) +* 3 when some source files could not be read (incomplete snapshot with remaining files created) Fatal errors occur for example when restic is unable to write to the backup destination, when there are network connectivity issues preventing successful communication, or when an invalid diff --git a/doc/REST_backend.rst b/doc/REST_backend.rst index f9d72cf06..9e85187f9 100644 --- a/doc/REST_backend.rst +++ b/doc/REST_backend.rst @@ -7,18 +7,18 @@ API. The following values are valid for ``{type}``: - * ``data`` - * ``keys`` - * ``locks`` - * ``snapshots`` - * ``index`` - * ``config`` +* ``data`` +* ``keys`` +* ``locks`` +* ``snapshots`` +* ``index`` +* ``config`` The API version is selected via the ``Accept`` HTTP header in the request. The following values are defined: - * ``application/vnd.x.restic.rest.v1`` or empty: Select API version 1 - * ``application/vnd.x.restic.rest.v2``: Select API version 2 +* ``application/vnd.x.restic.rest.v1`` or empty: Select API version 1 +* ``application/vnd.x.restic.rest.v2``: Select API version 2 The server will respond with the value of the highest version it supports in the ``Content-Type`` HTTP response header for the HTTP requests which should diff --git a/doc/design.rst b/doc/design.rst index 1e00a3358..b80029d10 100644 --- a/doc/design.rst +++ b/doc/design.rst @@ -824,4 +824,4 @@ Changes Repository Version 2 -------------------- - * Support compression for blobs (data/tree) and index / lock / snapshot files +* Support compression for blobs (data/tree) and index / lock / snapshot files diff --git a/doc/developer_information.rst b/doc/developer_information.rst index e7b60a03b..c7757e087 100644 --- a/doc/developer_information.rst +++ b/doc/developer_information.rst @@ -9,14 +9,14 @@ restic for version 0.10.0 and later. For restic versions down to 0.9.3 please refer to the documentation for the respective version. The binary produced depends on the following things: - * The source code for the release - * The exact version of the official `Go compiler `__ used to produce the binaries (running ``restic version`` will print this) - * The architecture and operating system the Go compiler runs on (Linux, ``amd64``) - * The build tags (for official binaries, it's the tag ``selfupdate``) - * The path where the source code is extracted to (``/restic``) - * The path to the Go compiler (``/usr/local/go``) - * The path to the Go workspace (``GOPATH=/home/build/go``) - * Other environment variables (mostly ``$GOOS``, ``$GOARCH``, ``$CGO_ENABLED``) +* The source code for the release +* The exact version of the official `Go compiler `__ used to produce the binaries (running ``restic version`` will print this) +* The architecture and operating system the Go compiler runs on (Linux, ``amd64``) +* The build tags (for official binaries, it's the tag ``selfupdate``) +* The path where the source code is extracted to (``/restic``) +* The path to the Go compiler (``/usr/local/go``) +* The path to the Go workspace (``GOPATH=/home/build/go``) +* Other environment variables (mostly ``$GOOS``, ``$GOARCH``, ``$CGO_ENABLED``) In addition, The compressed ZIP files for Windows depends on the modification timestamp and filename of the binary contained in it. In order to reproduce the @@ -69,9 +69,9 @@ container can be found in the `GitHub repository `__ The container serves the following goals: - * Have a very controlled environment which is independent from the local system - * Make it easy to have the correct version of the Go compiler at the right path - * Make it easy to pass in the source code to build at a well-defined path +* Have a very controlled environment which is independent from the local system +* Make it easy to have the correct version of the Go compiler at the right path +* Make it easy to pass in the source code to build at a well-defined path The following steps are necessary to build the binaries: diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index d1c64ba6e..1ccd9268d 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -428,10 +428,10 @@ This allows faster operations, since meta data does not need to be loaded from a remote repository. The cache is automatically created, usually in an OS-specific cache folder: - * Linux/other: ``$XDG_CACHE_HOME/restic``, or ``~/.cache/restic`` if - ``XDG_CACHE_HOME`` is not set - * macOS: ``~/Library/Caches/restic`` - * Windows: ``%LOCALAPPDATA%/restic`` +* Linux/other: ``$XDG_CACHE_HOME/restic``, or ``~/.cache/restic`` if + ``XDG_CACHE_HOME`` is not set +* macOS: ``~/Library/Caches/restic`` +* Windows: ``%LOCALAPPDATA%/restic`` If the relevant environment variables are not set, restic exits with an error message. From 6cc2bec5ddcd9bd2cec7dd825f4d3cb58b52d8b2 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Tue, 23 Jan 2024 19:09:04 +0100 Subject: [PATCH 050/158] apply suggestion from review --- changelog/unreleased/issue-4251 | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/changelog/unreleased/issue-4251 b/changelog/unreleased/issue-4251 index 5aeb50062..d1d3f4508 100644 --- a/changelog/unreleased/issue-4251 +++ b/changelog/unreleased/issue-4251 @@ -1,15 +1,16 @@ -Enhancement: Support reading backup from a program's standard output - -When reading data from stdin, the `backup` command could not verify whether the -corresponding command completed successfully. +Enhancement: Support reading backup from a commands's standard output The `backup` command now supports the `--stdin-from-command` option. When using -this option, the arguments to `backup` are interpreted as a command. `backup` -then executes the command and stores its standard output in the backup. This -enables restic to verify that the command completes with exit code zero. A -non-zero exit code causes the backup to fail. +this option, the arguments to `backup` are interpreted as a command instead of +paths to back up. `backup` then executes the given command and stores the +standard output from it in the backup, similar to the what the `--stdin` option +does. This also enables restic to verify that the command completes with exit +code zero. A non-zero exit code causes the backup to fail. -Example: `restic backup --stdin-from-command mysqldump [...]` +Note that the `--stdin` option does not have to be specified at the same time, +and that the `--stdin-filename` option also applies to `--stdin-from-command`. + +Example: `restic backup --stdin-from-command --stdin-filename dump.sql mysqldump [...]` https://github.com/restic/restic/issues/4251 https://github.com/restic/restic/pull/4410 From 66103aea3daa8d6455ecee8a2dae173e2f2d7b8c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 21 Jan 2024 17:43:13 +0100 Subject: [PATCH 051/158] Remove all usages of the global command-specific options Now, every command uses an options struct, which is passed to the run* function by the command.RunE method. --- cmd/restic/cmd_backup.go | 2 +- cmd/restic/cmd_debug.go | 46 ++++++++++++----------- cmd/restic/cmd_diff.go | 2 +- cmd/restic/cmd_forget.go | 7 ++-- cmd/restic/cmd_forget_integration_test.go | 5 ++- cmd/restic/cmd_generate.go | 28 +++++++------- cmd/restic/cmd_key.go | 38 ++++++++++--------- cmd/restic/cmd_key_integration_test.go | 25 ++++++------ cmd/restic/cmd_list.go | 6 +-- cmd/restic/cmd_list_integration_test.go | 2 +- cmd/restic/cmd_ls.go | 2 +- cmd/restic/cmd_prune.go | 4 +- cmd/restic/cmd_prune_integration_test.go | 5 ++- 13 files changed, 93 insertions(+), 79 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index a2b81a759..4b27bb83c 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -633,7 +633,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter wg.Go(func() error { return sc.Scan(cancelCtx, targets) }) } - arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: backupOptions.ReadConcurrency}) + arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: opts.ReadConcurrency}) arch.SelectByName = selectByNameFilter arch.Select = selectFilter arch.WithAtime = opts.WithAtime diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 60413de21..49efb904b 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -52,19 +52,23 @@ Exit status is 0 if the command was successful, and non-zero if there was any er }, } -var tryRepair bool -var repairByte bool -var extractPack bool -var reuploadBlobs bool +type DebugExamineOptions struct { + TryRepair bool + RepairByte bool + ExtractPack bool + ReuploadBlobs bool +} + +var debugExamineOpts DebugExamineOptions func init() { cmdRoot.AddCommand(cmdDebug) cmdDebug.AddCommand(cmdDebugDump) cmdDebug.AddCommand(cmdDebugExamine) - cmdDebugExamine.Flags().BoolVar(&extractPack, "extract-pack", false, "write blobs to the current directory") - cmdDebugExamine.Flags().BoolVar(&reuploadBlobs, "reupload-blobs", false, "reupload blobs to the repository") - cmdDebugExamine.Flags().BoolVar(&tryRepair, "try-repair", false, "try to repair broken blobs with single bit flips") - cmdDebugExamine.Flags().BoolVar(&repairByte, "repair-byte", false, "try to repair broken blobs by trying bytes") + cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.ExtractPack, "extract-pack", false, "write blobs to the current directory") + cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.ReuploadBlobs, "reupload-blobs", false, "reupload blobs to the repository") + cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.TryRepair, "try-repair", false, "try to repair broken blobs with single bit flips") + cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.RepairByte, "repair-byte", false, "try to repair broken blobs by trying bytes") } func prettyPrintJSON(wr io.Writer, item interface{}) error { @@ -196,7 +200,7 @@ var cmdDebugExamine = &cobra.Command{ Short: "Examine a pack file", DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runDebugExamine(cmd.Context(), globalOptions, args) + return runDebugExamine(cmd.Context(), globalOptions, debugExamineOpts, args) }, } @@ -315,7 +319,7 @@ func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte { return out } -func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, list []restic.Blob) error { +func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, packID restic.ID, list []restic.Blob) error { dec, err := zstd.NewReader(nil) if err != nil { panic(err) @@ -328,7 +332,7 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li wg, ctx := errgroup.WithContext(ctx) - if reuploadBlobs { + if opts.ReuploadBlobs { repo.StartPackUploader(ctx, wg) } @@ -356,8 +360,8 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li filePrefix := "" if err != nil { Warnf("error decrypting blob: %v\n", err) - if tryRepair || repairByte { - plaintext = tryRepairWithBitflip(ctx, key, buf, repairByte) + if opts.TryRepair || opts.RepairByte { + plaintext = tryRepairWithBitflip(ctx, key, buf, opts.RepairByte) } if plaintext != nil { outputPrefix = "repaired " @@ -391,13 +395,13 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li Printf(" successfully %vdecrypted blob (length %v), hash is %v, ID matches\n", outputPrefix, len(plaintext), id) prefix = "correct-" } - if extractPack { + if opts.ExtractPack { err = storePlainBlob(id, filePrefix+prefix, plaintext) if err != nil { return err } } - if reuploadBlobs { + if opts.ReuploadBlobs { _, _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true) if err != nil { return err @@ -406,7 +410,7 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li } } - if reuploadBlobs { + if opts.ReuploadBlobs { return repo.Flush(ctx) } return nil @@ -437,7 +441,7 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte) error { return nil } -func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) error { +func runDebugExamine(ctx context.Context, gopts GlobalOptions, opts DebugExamineOptions, args []string) error { repo, err := OpenRepository(ctx, gopts) if err != nil { return err @@ -476,7 +480,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er } for _, id := range ids { - err := examinePack(ctx, repo, id) + err := examinePack(ctx, opts, repo, id) if err != nil { Warnf("error: %v\n", err) } @@ -487,7 +491,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er return nil } -func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) error { +func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID) error { Printf("examine %v\n", id) h := backend.Handle{ @@ -524,7 +528,7 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro checkPackSize(blobs, fi.Size) - err = loadBlobs(ctx, repo, id, blobs) + err = loadBlobs(ctx, opts, repo, id, blobs) if err != nil { Warnf("error: %v\n", err) } else { @@ -542,7 +546,7 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro checkPackSize(blobs, fi.Size) if !blobsLoaded { - return loadBlobs(ctx, repo, id, blobs) + return loadBlobs(ctx, opts, repo, id, blobs) } return nil } diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index ea40d2860..7878613e1 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -401,7 +401,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args [] c := &Comparer{ repo: repo, - opts: diffOptions, + opts: opts, printChange: func(change *Change) { Printf("%-5s%v\n", change.Modifier, change.Path) }, diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index a7f39dc4e..65ff449a3 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -33,7 +33,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runForget(cmd.Context(), forgetOptions, globalOptions, args) + return runForget(cmd.Context(), forgetOptions, forgetPruneOptions, globalOptions, args) }, } @@ -98,6 +98,7 @@ type ForgetOptions struct { } var forgetOptions ForgetOptions +var forgetPruneOptions PruneOptions func init() { cmdRoot.AddCommand(cmdForget) @@ -132,7 +133,7 @@ func init() { f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed") f.SortFlags = false - addPruneOptions(cmdForget) + addPruneOptions(cmdForget, &forgetPruneOptions) } func verifyForgetOptions(opts *ForgetOptions) error { @@ -151,7 +152,7 @@ func verifyForgetOptions(opts *ForgetOptions) error { return nil } -func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, args []string) error { +func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOptions, gopts GlobalOptions, args []string) error { err := verifyForgetOptions(&opts) if err != nil { return err diff --git a/cmd/restic/cmd_forget_integration_test.go b/cmd/restic/cmd_forget_integration_test.go index 8908d5a5f..1c027a240 100644 --- a/cmd/restic/cmd_forget_integration_test.go +++ b/cmd/restic/cmd_forget_integration_test.go @@ -9,5 +9,8 @@ import ( func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) { opts := ForgetOptions{} - rtest.OK(t, runForget(context.TODO(), opts, gopts, args)) + pruneOpts := PruneOptions{ + MaxUnused: "5%", + } + rtest.OK(t, runForget(context.TODO(), opts, pruneOpts, gopts, args)) } diff --git a/cmd/restic/cmd_generate.go b/cmd/restic/cmd_generate.go index b284767ca..9d1652e93 100644 --- a/cmd/restic/cmd_generate.go +++ b/cmd/restic/cmd_generate.go @@ -21,7 +21,9 @@ EXIT STATUS Exit status is 0 if the command was successful, and non-zero if there was any error. `, DisableAutoGenTag: true, - RunE: runGenerate, + RunE: func(cmd *cobra.Command, args []string) error { + return runGenerate(genOpts, args) + }, } type generateOptions struct { @@ -90,48 +92,48 @@ func writePowerShellCompletion(file string) error { return cmdRoot.GenPowerShellCompletionFile(file) } -func runGenerate(_ *cobra.Command, args []string) error { +func runGenerate(opts generateOptions, args []string) error { if len(args) > 0 { return errors.Fatal("the generate command expects no arguments, only options - please see `restic help generate` for usage and flags") } - if genOpts.ManDir != "" { - err := writeManpages(genOpts.ManDir) + if opts.ManDir != "" { + err := writeManpages(opts.ManDir) if err != nil { return err } } - if genOpts.BashCompletionFile != "" { - err := writeBashCompletion(genOpts.BashCompletionFile) + if opts.BashCompletionFile != "" { + err := writeBashCompletion(opts.BashCompletionFile) if err != nil { return err } } - if genOpts.FishCompletionFile != "" { - err := writeFishCompletion(genOpts.FishCompletionFile) + if opts.FishCompletionFile != "" { + err := writeFishCompletion(opts.FishCompletionFile) if err != nil { return err } } - if genOpts.ZSHCompletionFile != "" { - err := writeZSHCompletion(genOpts.ZSHCompletionFile) + if opts.ZSHCompletionFile != "" { + err := writeZSHCompletion(opts.ZSHCompletionFile) if err != nil { return err } } - if genOpts.PowerShellCompletionFile != "" { - err := writePowerShellCompletion(genOpts.PowerShellCompletionFile) + if opts.PowerShellCompletionFile != "" { + err := writePowerShellCompletion(opts.PowerShellCompletionFile) if err != nil { return err } } var empty generateOptions - if genOpts == empty { + if opts == empty { return errors.Fatal("nothing to do, please specify at least one output file/dir") } diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index e147f537e..8309f8382 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -29,23 +29,25 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runKey(cmd.Context(), globalOptions, args) + return runKey(cmd.Context(), globalOptions, keyOpts, args) }, } -var ( - newPasswordFile string - keyUsername string - keyHostname string -) +type KeyOptions struct { + NewPasswordFile string + Username string + Hostname string +} + +var keyOpts KeyOptions func init() { cmdRoot.AddCommand(cmdKey) flags := cmdKey.Flags() - flags.StringVarP(&newPasswordFile, "new-password-file", "", "", "`file` from which to read the new password") - flags.StringVarP(&keyUsername, "user", "", "", "the username for new keys") - flags.StringVarP(&keyHostname, "host", "", "", "the hostname for new keys") + flags.StringVarP(&keyOpts.NewPasswordFile, "new-password-file", "", "", "`file` from which to read the new password") + flags.StringVarP(&keyOpts.Username, "user", "", "", "the username for new keys") + flags.StringVarP(&keyOpts.Hostname, "host", "", "", "the hostname for new keys") } func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions) error { @@ -105,7 +107,7 @@ func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions // testKeyNewPassword is used to set a new password during integration testing. var testKeyNewPassword string -func getNewPassword(gopts GlobalOptions) (string, error) { +func getNewPassword(gopts GlobalOptions, newPasswordFile string) (string, error) { if testKeyNewPassword != "" { return testKeyNewPassword, nil } @@ -124,13 +126,13 @@ func getNewPassword(gopts GlobalOptions) (string, error) { "enter password again: ") } -func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error { - pw, err := getNewPassword(gopts) +func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyOptions) error { + pw, err := getNewPassword(gopts, opts.NewPasswordFile) if err != nil { return err } - id, err := repository.AddKey(ctx, repo, pw, keyUsername, keyHostname, repo.Key()) + id, err := repository.AddKey(ctx, repo, pw, opts.Username, opts.Hostname, repo.Key()) if err != nil { return errors.Fatalf("creating new key failed: %v\n", err) } @@ -160,8 +162,8 @@ func deleteKey(ctx context.Context, repo *repository.Repository, id restic.ID) e return nil } -func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error { - pw, err := getNewPassword(gopts) +func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, newPasswordFile string) error { + pw, err := getNewPassword(gopts, newPasswordFile) if err != nil { return err } @@ -201,7 +203,7 @@ func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repos return nil } -func runKey(ctx context.Context, gopts GlobalOptions, args []string) error { +func runKey(ctx context.Context, gopts GlobalOptions, opts KeyOptions, args []string) error { if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) { return errors.Fatal("wrong number of arguments") } @@ -230,7 +232,7 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error { return err } - return addKey(ctx, repo, gopts) + return addKey(ctx, repo, gopts, opts) case "remove": lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) defer unlockRepo(lock) @@ -251,7 +253,7 @@ func runKey(ctx context.Context, gopts GlobalOptions, args []string) error { return err } - return changePassword(ctx, repo, gopts) + return changePassword(ctx, repo, gopts, opts.NewPasswordFile) } return nil diff --git a/cmd/restic/cmd_key_integration_test.go b/cmd/restic/cmd_key_integration_test.go index f68799dde..34474c3af 100644 --- a/cmd/restic/cmd_key_integration_test.go +++ b/cmd/restic/cmd_key_integration_test.go @@ -13,7 +13,7 @@ import ( func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string { buf, err := withCaptureStdout(func() error { - return runKey(context.TODO(), gopts, []string{"list"}) + return runKey(context.TODO(), gopts, KeyOptions{}, []string{"list"}) }) rtest.OK(t, err) @@ -36,21 +36,20 @@ func testRunKeyAddNewKey(t testing.TB, newPassword string, gopts GlobalOptions) testKeyNewPassword = "" }() - rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"})) + rtest.OK(t, runKey(context.TODO(), gopts, KeyOptions{}, []string{"add"})) } func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) { testKeyNewPassword = "john's geheimnis" defer func() { testKeyNewPassword = "" - keyUsername = "" - keyHostname = "" }() - rtest.OK(t, cmdKey.Flags().Parse([]string{"--user=john", "--host=example.com"})) - t.Log("adding key for john@example.com") - rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"})) + rtest.OK(t, runKey(context.TODO(), gopts, KeyOptions{ + Username: "john", + Hostname: "example.com", + }, []string{"add"})) repo, err := OpenRepository(context.TODO(), gopts) rtest.OK(t, err) @@ -67,13 +66,13 @@ func testRunKeyPasswd(t testing.TB, newPassword string, gopts GlobalOptions) { testKeyNewPassword = "" }() - rtest.OK(t, runKey(context.TODO(), gopts, []string{"passwd"})) + rtest.OK(t, runKey(context.TODO(), gopts, KeyOptions{}, []string{"passwd"})) } func testRunKeyRemove(t testing.TB, gopts GlobalOptions, IDs []string) { t.Logf("remove %d keys: %q\n", len(IDs), IDs) for _, id := range IDs { - rtest.OK(t, runKey(context.TODO(), gopts, []string{"remove", id})) + rtest.OK(t, runKey(context.TODO(), gopts, KeyOptions{}, []string{"remove", id})) } } @@ -103,7 +102,7 @@ func TestKeyAddRemove(t *testing.T) { env.gopts.password = passwordList[len(passwordList)-1] t.Logf("testing access with last password %q\n", env.gopts.password) - rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"})) + rtest.OK(t, runKey(context.TODO(), env.gopts, KeyOptions{}, []string{"list"})) testRunCheck(t, env.gopts) testRunKeyAddNewKeyUserHost(t, env.gopts) @@ -131,15 +130,15 @@ func TestKeyProblems(t *testing.T) { testKeyNewPassword = "" }() - err := runKey(context.TODO(), env.gopts, []string{"passwd"}) + err := runKey(context.TODO(), env.gopts, KeyOptions{}, []string{"passwd"}) t.Log(err) rtest.Assert(t, err != nil, "expected passwd change to fail") - err = runKey(context.TODO(), env.gopts, []string{"add"}) + err = runKey(context.TODO(), env.gopts, KeyOptions{}, []string{"add"}) t.Log(err) rtest.Assert(t, err != nil, "expected key adding to fail") t.Logf("testing access with initial password %q\n", env.gopts.password) - rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"})) + rtest.OK(t, runKey(context.TODO(), env.gopts, KeyOptions{}, []string{"list"})) testRunCheck(t, env.gopts) } diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go index 38f8b094a..8be99234f 100644 --- a/cmd/restic/cmd_list.go +++ b/cmd/restic/cmd_list.go @@ -23,7 +23,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runList(cmd.Context(), cmd, globalOptions, args) + return runList(cmd.Context(), globalOptions, args) }, } @@ -31,9 +31,9 @@ func init() { cmdRoot.AddCommand(cmdList) } -func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args []string) error { +func runList(ctx context.Context, gopts GlobalOptions, args []string) error { if len(args) != 1 { - return errors.Fatal("type not specified, usage: " + cmd.Use) + return errors.Fatal("type not specified") } repo, err := OpenRepository(ctx, gopts) diff --git a/cmd/restic/cmd_list_integration_test.go b/cmd/restic/cmd_list_integration_test.go index 4140a3ea8..ef2b8bf8f 100644 --- a/cmd/restic/cmd_list_integration_test.go +++ b/cmd/restic/cmd_list_integration_test.go @@ -12,7 +12,7 @@ import ( func testRunList(t testing.TB, tpe string, opts GlobalOptions) restic.IDs { buf, err := withCaptureStdout(func() error { - return runList(context.TODO(), cmdList, opts, []string{tpe}) + return runList(context.TODO(), opts, []string{tpe}) }) rtest.OK(t, err) return parseIDsFromReader(t, buf) diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 07c49d60f..d30e2819c 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -210,7 +210,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri Verbosef("%v filtered by %v:\n", sn, dirs) } printNode = func(path string, node *restic.Node) { - Printf("%s\n", formatNode(path, node, lsOptions.ListLong, lsOptions.HumanReadable)) + Printf("%s\n", formatNode(path, node, opts.ListLong, opts.HumanReadable)) } } diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index 739a450df..10abbf9f0 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -66,10 +66,10 @@ func init() { f := cmdPrune.Flags() f.BoolVarP(&pruneOptions.DryRun, "dry-run", "n", false, "do not modify the repository, just print what would be done") f.StringVarP(&pruneOptions.UnsafeNoSpaceRecovery, "unsafe-recover-no-free-space", "", "", "UNSAFE, READ THE DOCUMENTATION BEFORE USING! Try to recover a repository stuck with no free space. Do not use without trying out 'prune --max-repack-size 0' first.") - addPruneOptions(cmdPrune) + addPruneOptions(cmdPrune, &pruneOptions) } -func addPruneOptions(c *cobra.Command) { +func addPruneOptions(c *cobra.Command, pruneOptions *PruneOptions) { f := c.Flags() f.StringVar(&pruneOptions.MaxUnused, "max-unused", "5%", "tolerate given `limit` of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited')") f.StringVar(&pruneOptions.MaxRepackSize, "max-repack-size", "", "maximum `size` to repack (allowed suffixes: k/K, m/M, g/G, t/T)") diff --git a/cmd/restic/cmd_prune_integration_test.go b/cmd/restic/cmd_prune_integration_test.go index 53e27ee10..ebfa7ae4e 100644 --- a/cmd/restic/cmd_prune_integration_test.go +++ b/cmd/restic/cmd_prune_integration_test.go @@ -81,7 +81,10 @@ func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) { DryRun: true, Last: 1, } - return runForget(context.TODO(), opts, gopts, args) + pruneOpts := PruneOptions{ + MaxUnused: "5%", + } + return runForget(context.TODO(), opts, pruneOpts, gopts, args) }) rtest.OK(t, err) From ae9683336d04f694c188511b4dffe76f581e335f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 26 Jan 2024 23:51:54 +0100 Subject: [PATCH 052/158] CONTRIBUTING.md: Clarify handling of small bugfixes Opening an issue for a small bugfix is usually not useful. It primarily adds overhead. --- CONTRIBUTING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4318a2107..dc278fa3a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,8 @@ Ways to Help Out Thank you for your contribution! Please **open an issue first** (or add a comment to an existing issue) if you plan to work on any code or add a new feature. This way, duplicate work is prevented and we can discuss your ideas -and design first. +and design first. Small bugfixes are an exception to this rule, just open a +pull request in this case. There are several ways you can help us out. First of all code contributions and bug fixes are most welcome. However even "minor" details as fixing spelling From c13bf0b6078a1fea8b0e760c4ac8fbb7e8258a3b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 21 Jan 2024 20:38:42 +0100 Subject: [PATCH 053/158] repository: Introduce RemoveKey function This replaces directly removing keys via the backend. --- cmd/restic/cmd_key.go | 10 +++------- internal/repository/key.go | 9 +++++++++ internal/repository/repository.go | 13 ++++++++++--- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index e147f537e..0952a423b 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -7,7 +7,6 @@ import ( "strings" "sync" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" @@ -150,8 +149,7 @@ func deleteKey(ctx context.Context, repo *repository.Repository, id restic.ID) e return errors.Fatal("refusing to remove key currently used to access repository") } - h := backend.Handle{Type: restic.KeyFile, Name: id.String()} - err := repo.Backend().Remove(ctx, h) + err := repository.RemoveKey(ctx, repo, id) if err != nil { return err } @@ -177,8 +175,7 @@ func changePassword(ctx context.Context, repo *repository.Repository, gopts Glob return err } - h := backend.Handle{Type: restic.KeyFile, Name: oldID.String()} - err = repo.Backend().Remove(ctx, h) + err = repository.RemoveKey(ctx, repo, oldID) if err != nil { return err } @@ -194,8 +191,7 @@ func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repos err := repo.SearchKey(ctx, pw, 0, key.ID().String()) if err != nil { // the key is invalid, try to remove it - h := backend.Handle{Type: restic.KeyFile, Name: key.ID().String()} - _ = repo.Backend().Remove(ctx, h) + _ = repository.RemoveKey(ctx, repo, key.ID()) return errors.Fatalf("failed to access repository with new key: %v", err) } return nil diff --git a/internal/repository/key.go b/internal/repository/key.go index 638d15d91..5f7a93297 100644 --- a/internal/repository/key.go +++ b/internal/repository/key.go @@ -285,6 +285,15 @@ func AddKey(ctx context.Context, s *Repository, password, username, hostname str return newkey, nil } +func RemoveKey(ctx context.Context, repo *Repository, id restic.ID) error { + if id == repo.KeyID() { + return errors.New("refusing to remove key currently used to access repository") + } + + h := backend.Handle{Type: restic.KeyFile, Name: id.String()} + return repo.be.Remove(ctx, h) +} + func (k *Key) String() string { if k == nil { return "" diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 407b6429c..9db83a4df 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -743,12 +743,19 @@ func (r *Repository) SearchKey(ctx context.Context, password string, maxKeys int return err } + oldKey := r.key + oldKeyID := r.keyID + r.key = key.master r.keyID = key.ID() cfg, err := restic.LoadConfig(ctx, r) - if err == crypto.ErrUnauthenticated { - return fmt.Errorf("config or key %v is damaged: %w", key.ID(), err) - } else if err != nil { + if err != nil { + r.key = oldKey + r.keyID = oldKeyID + + if err == crypto.ErrUnauthenticated { + return fmt.Errorf("config or key %v is damaged: %w", key.ID(), err) + } return fmt.Errorf("config cannot be loaded: %w", err) } From bedff1ed6d7402759ba57d4de4b8c5fc80889dfb Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 15:40:09 +0100 Subject: [PATCH 054/158] split deleteFiles into UI and logic parts --- cmd/restic/delete.go | 56 ++++++++++--------------------------- internal/restic/parallel.go | 42 ++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 42 deletions(-) diff --git a/cmd/restic/delete.go b/cmd/restic/delete.go index 1b7937bd3..c3a7e039d 100644 --- a/cmd/restic/delete.go +++ b/cmd/restic/delete.go @@ -3,9 +3,6 @@ package main import ( "context" - "golang.org/x/sync/errgroup" - - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/restic" ) @@ -24,46 +21,21 @@ func DeleteFilesChecked(ctx context.Context, gopts GlobalOptions, repo restic.Re // deleteFiles deletes the given fileList of fileType in parallel // if ignoreError=true, it will print a warning if there was an error, else it will abort. func deleteFiles(ctx context.Context, gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error { - totalCount := len(fileList) - fileChan := make(chan restic.ID) - wg, ctx := errgroup.WithContext(ctx) - wg.Go(func() error { - defer close(fileChan) - for id := range fileList { - select { - case fileChan <- id: - case <-ctx.Done(): - return ctx.Err() + bar := newProgressMax(!gopts.JSON && !gopts.Quiet, 0, "files deleted") + defer bar.Done() + + return restic.ParallelRemove(ctx, repo, fileList, fileType, func(id restic.ID, err error) error { + if err != nil { + if !gopts.JSON { + Warnf("unable to remove %v/%v from the repository\n", fileType, id) + } + if !ignoreError { + return err } } + if !gopts.JSON && gopts.verbosity > 2 { + Verbosef("removed %v/%v\n", fileType, id) + } return nil - }) - - bar := newProgressMax(!gopts.JSON && !gopts.Quiet, uint64(totalCount), "files deleted") - defer bar.Done() - // deleting files is IO-bound - workerCount := repo.Connections() - for i := 0; i < int(workerCount); i++ { - wg.Go(func() error { - for id := range fileChan { - h := backend.Handle{Type: fileType, Name: id.String()} - err := repo.Backend().Remove(ctx, h) - if err != nil { - if !gopts.JSON { - Warnf("unable to remove %v from the repository\n", h) - } - if !ignoreError { - return err - } - } - if !gopts.JSON && gopts.verbosity > 2 { - Verbosef("removed %v\n", h) - } - bar.Add(1) - } - return nil - }) - } - err := wg.Wait() - return err + }, bar) } diff --git a/internal/restic/parallel.go b/internal/restic/parallel.go index b22a249fe..cefbf0358 100644 --- a/internal/restic/parallel.go +++ b/internal/restic/parallel.go @@ -3,7 +3,9 @@ package restic import ( "context" + "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/debug" + "github.com/restic/restic/internal/ui/progress" "golang.org/x/sync/errgroup" ) @@ -50,3 +52,43 @@ func ParallelList(ctx context.Context, r Lister, t FileType, parallelism uint, f return wg.Wait() } + +// ParallelRemove deletes the given fileList of fileType in parallel +// if callback returns an error, then it will abort. +func ParallelRemove(ctx context.Context, repo Repository, fileList IDSet, fileType FileType, report func(id ID, err error) error, bar *progress.Counter) error { + fileChan := make(chan ID) + wg, ctx := errgroup.WithContext(ctx) + wg.Go(func() error { + defer close(fileChan) + for id := range fileList { + select { + case fileChan <- id: + case <-ctx.Done(): + return ctx.Err() + } + } + return nil + }) + + bar.SetMax(uint64(len(fileList))) + + // deleting files is IO-bound + workerCount := repo.Connections() + for i := 0; i < int(workerCount); i++ { + wg.Go(func() error { + for id := range fileChan { + h := backend.Handle{Type: fileType, Name: id.String()} + err := repo.Backend().Remove(ctx, h) + if report != nil { + err = report(id, err) + } + if err != nil { + return err + } + bar.Add(1) + } + return nil + }) + } + return wg.Wait() +} From cb50832d508906be8a5845519147f01b921154df Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 15:58:06 +0100 Subject: [PATCH 055/158] index: let MasterIndex.Save also delete obsolete indexes --- cmd/restic/cmd_prune.go | 32 +++++++++--------- cmd/restic/cmd_repair_index.go | 2 +- cmd/restic/cmd_repair_packs.go | 2 +- internal/index/master_index.go | 50 +++++++++++++++++++---------- internal/index/master_index_test.go | 12 +------ internal/repository/repack_test.go | 30 ++++++----------- internal/restic/repository.go | 9 +++++- 7 files changed, 69 insertions(+), 68 deletions(-) diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index 10abbf9f0..efd8f6e3a 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -15,6 +15,7 @@ import ( "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/ui" + "github.com/restic/restic/internal/ui/progress" "github.com/spf13/cobra" ) @@ -766,7 +767,7 @@ func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo r return errors.Fatalf("%s", err) } } else if len(plan.ignorePacks) != 0 { - err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil) + err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil, false) if err != nil { return errors.Fatalf("%s", err) } @@ -778,7 +779,7 @@ func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo r } if opts.unsafeRecovery { - _, err = writeIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil) + err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil, true) if err != nil { return errors.Fatalf("%s", err) } @@ -788,23 +789,22 @@ func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo r return nil } -func writeIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) (restic.IDSet, error) { +func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs, skipDeletion bool) error { Verbosef("rebuilding index\n") bar := newProgressMax(!gopts.Quiet, 0, "packs processed") - obsoleteIndexes, err := repo.Index().Save(ctx, repo, removePacks, extraObsolete, bar) - bar.Done() - return obsoleteIndexes, err -} - -func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error { - obsoleteIndexes, err := writeIndexFiles(ctx, gopts, repo, removePacks, extraObsolete) - if err != nil { - return err - } - - Verbosef("deleting obsolete index files\n") - return DeleteFilesChecked(ctx, gopts, repo, obsoleteIndexes, restic.IndexFile) + return repo.Index().Save(ctx, repo, removePacks, extraObsolete, restic.MasterIndexSaveOpts{ + SaveProgress: bar, + DeleteProgress: func() *progress.Counter { + return newProgressMax(!gopts.Quiet, 0, "old indexes deleted") + }, + DeleteReport: func(id restic.ID, err error) { + if gopts.verbosity > 2 { + Verbosef("removed index %v\n", id.String()) + } + }, + SkipDeletion: skipDeletion, + }) } func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (usedBlobs restic.CountedBlobSet, err error) { diff --git a/cmd/restic/cmd_repair_index.go b/cmd/restic/cmd_repair_index.go index c8a94b470..fc5506b34 100644 --- a/cmd/restic/cmd_repair_index.go +++ b/cmd/restic/cmd_repair_index.go @@ -154,7 +154,7 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti } } - err = rebuildIndexFiles(ctx, gopts, repo, removePacks, obsoleteIndexes) + err = rebuildIndexFiles(ctx, gopts, repo, removePacks, obsoleteIndexes, false) if err != nil { return err } diff --git a/cmd/restic/cmd_repair_packs.go b/cmd/restic/cmd_repair_packs.go index 723bdbccb..c572e02c5 100644 --- a/cmd/restic/cmd_repair_packs.go +++ b/cmd/restic/cmd_repair_packs.go @@ -145,7 +145,7 @@ func repairPacks(ctx context.Context, gopts GlobalOptions, repo *repository.Repo bar.Done() // remove salvaged packs from index - err = rebuildIndexFiles(ctx, gopts, repo, ids, nil) + err = rebuildIndexFiles(ctx, gopts, repo, ids, nil, false) if err != nil { return errors.Fatalf("%s", err) } diff --git a/internal/index/master_index.go b/internal/index/master_index.go index 073c9ace4..4c114b955 100644 --- a/internal/index/master_index.go +++ b/internal/index/master_index.go @@ -9,7 +9,6 @@ import ( "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/ui/progress" "golang.org/x/sync/errgroup" ) @@ -267,23 +266,22 @@ func (mi *MasterIndex) MergeFinalIndexes() error { // Save saves all known indexes to index files, leaving out any // packs whose ID is contained in packBlacklist from finalized indexes. -// The new index contains the IDs of all known indexes in the "supersedes" -// field. The IDs are also returned in the IDSet obsolete. -// After calling this function, you should remove the obsolete index files. -func (mi *MasterIndex) Save(ctx context.Context, repo restic.SaverUnpacked, packBlacklist restic.IDSet, extraObsolete restic.IDs, p *progress.Counter) (obsolete restic.IDSet, err error) { - p.SetMax(uint64(len(mi.Packs(packBlacklist)))) +// It also removes the old index files and those listed in extraObsolete. +func (mi *MasterIndex) Save(ctx context.Context, repo restic.Repository, excludePacks restic.IDSet, extraObsolete restic.IDs, opts restic.MasterIndexSaveOpts) error { + p := opts.SaveProgress + p.SetMax(uint64(len(mi.Packs(excludePacks)))) mi.idxMutex.Lock() defer mi.idxMutex.Unlock() - debug.Log("start rebuilding index of %d indexes, pack blacklist: %v", len(mi.idx), packBlacklist) + debug.Log("start rebuilding index of %d indexes, excludePacks: %v", len(mi.idx), excludePacks) newIndex := NewIndex() - obsolete = restic.NewIDSet() + obsolete := restic.NewIDSet() // track spawned goroutines using wg, create a new context which is // cancelled as soon as an error occurs. - wg, ctx := errgroup.WithContext(ctx) + wg, wgCtx := errgroup.WithContext(ctx) ch := make(chan *Index) @@ -310,21 +308,21 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.SaverUnpacked, pack debug.Log("adding index %d", i) - for pbs := range idx.EachByPack(ctx, packBlacklist) { + for pbs := range idx.EachByPack(wgCtx, excludePacks) { newIndex.StorePack(pbs.PackID, pbs.Blobs) p.Add(1) if IndexFull(newIndex, mi.compress) { select { case ch <- newIndex: - case <-ctx.Done(): - return ctx.Err() + case <-wgCtx.Done(): + return wgCtx.Err() } newIndex = NewIndex() } } } - err = newIndex.AddToSupersedes(extraObsolete...) + err := newIndex.AddToSupersedes(extraObsolete...) if err != nil { return err } @@ -332,7 +330,7 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.SaverUnpacked, pack select { case ch <- newIndex: - case <-ctx.Done(): + case <-wgCtx.Done(): } return nil }) @@ -341,7 +339,7 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.SaverUnpacked, pack worker := func() error { for idx := range ch { idx.Finalize() - if _, err := SaveIndex(ctx, repo, idx); err != nil { + if _, err := SaveIndex(wgCtx, repo, idx); err != nil { return err } } @@ -354,9 +352,27 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.SaverUnpacked, pack for i := 0; i < workerCount; i++ { wg.Go(worker) } - err = wg.Wait() + err := wg.Wait() + p.Done() + if err != nil { + return err + } - return obsolete, err + if opts.SkipDeletion { + return nil + } + + p = nil + if opts.DeleteProgress != nil { + p = opts.DeleteProgress() + } + defer p.Done() + return restic.ParallelRemove(ctx, repo, obsolete, restic.IndexFile, func(id restic.ID, err error) error { + if opts.DeleteReport != nil { + opts.DeleteReport(id, err) + } + return err + }, p) } // SaveIndex saves an index in the repository. diff --git a/internal/index/master_index_test.go b/internal/index/master_index_test.go index f76feb5fa..dcf6a94f6 100644 --- a/internal/index/master_index_test.go +++ b/internal/index/master_index_test.go @@ -8,7 +8,6 @@ import ( "testing" "time" - "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/checker" "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/index" @@ -363,20 +362,11 @@ func testIndexSave(t *testing.T, version uint) { t.Fatal(err) } - obsoletes, err := repo.Index().Save(context.TODO(), repo, nil, nil, nil) + err = repo.Index().Save(context.TODO(), repo, nil, nil, restic.MasterIndexSaveOpts{}) if err != nil { t.Fatalf("unable to save new index: %v", err) } - for id := range obsoletes { - t.Logf("remove index %v", id.Str()) - h := backend.Handle{Type: restic.IndexFile, Name: id.String()} - err = repo.Backend().Remove(context.TODO(), h) - if err != nil { - t.Errorf("error removing index %v: %v", id, err) - } - } - checker := checker.New(repo, false) err = checker.LoadSnapshots(context.TODO()) if err != nil { diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index 20f0f2685..bab04f6b7 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -173,39 +173,27 @@ func flush(t *testing.T, repo restic.Repository) { func rebuildIndex(t *testing.T, repo restic.Repository) { err := repo.SetIndex(index.NewMasterIndex()) - if err != nil { - t.Fatal(err) - } + rtest.OK(t, err) packs := make(map[restic.ID]int64) err = repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { packs[id] = size return nil }) - if err != nil { - t.Fatal(err) - } + rtest.OK(t, err) _, err = repo.(*repository.Repository).CreateIndexFromPacks(context.TODO(), packs, nil) - if err != nil { - t.Fatal(err) - } + rtest.OK(t, err) + var obsoleteIndexes restic.IDs err = repo.List(context.TODO(), restic.IndexFile, func(id restic.ID, size int64) error { - h := backend.Handle{ - Type: restic.IndexFile, - Name: id.String(), - } - return repo.Backend().Remove(context.TODO(), h) + obsoleteIndexes = append(obsoleteIndexes, id) + return nil }) - if err != nil { - t.Fatal(err) - } + rtest.OK(t, err) - _, err = repo.Index().Save(context.TODO(), repo, restic.NewIDSet(), nil, nil) - if err != nil { - t.Fatal(err) - } + err = repo.Index().Save(context.TODO(), repo, restic.NewIDSet(), obsoleteIndexes, restic.MasterIndexSaveOpts{}) + rtest.OK(t, err) } func reloadIndex(t *testing.T, repo restic.Repository) { diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 6818847c0..1c6c8d39d 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -89,6 +89,13 @@ type PackBlobs struct { Blobs []Blob } +type MasterIndexSaveOpts struct { + SaveProgress *progress.Counter + DeleteProgress func() *progress.Counter + DeleteReport func(id ID, err error) + SkipDeletion bool +} + // MasterIndex keeps track of the blobs are stored within files. type MasterIndex interface { Has(BlobHandle) bool @@ -99,7 +106,7 @@ type MasterIndex interface { Each(ctx context.Context, fn func(PackedBlob)) ListPacks(ctx context.Context, packs IDSet) <-chan PackBlobs - Save(ctx context.Context, repo SaverUnpacked, packBlacklist IDSet, extraObsolete IDs, p *progress.Counter) (obsolete IDSet, err error) + Save(ctx context.Context, repo Repository, excludePacks IDSet, extraObsolete IDs, opts MasterIndexSaveOpts) error } // Lister allows listing files in a backend. From d26d2d41f82eb35e263d16d1ad0241e12b4ccb8f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 18:10:11 +0100 Subject: [PATCH 056/158] backup/restore: extract termstatus initialization --- cmd/restic/cmd_backup.go | 30 +++------------------------- cmd/restic/cmd_restore.go | 29 +++------------------------ cmd/restic/termstatus.go | 42 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 53 deletions(-) create mode 100644 cmd/restic/termstatus.go diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 348050895..0596ee918 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -12,7 +12,6 @@ import ( "runtime" "strconv" "strings" - "sync" "time" "github.com/spf13/cobra" @@ -25,7 +24,6 @@ import ( "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/textfile" - "github.com/restic/restic/internal/ui" "github.com/restic/restic/internal/ui/backup" "github.com/restic/restic/internal/ui/termstatus" ) @@ -56,31 +54,9 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea }, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - var wg sync.WaitGroup - cancelCtx, cancel := context.WithCancel(ctx) - defer func() { - // shutdown termstatus - cancel() - wg.Wait() - }() - - term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet) - wg.Add(1) - go func() { - defer wg.Done() - term.Run(cancelCtx) - }() - - // use the terminal for stdout/stderr - prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr - defer func() { - globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr - }() - stdioWrapper := ui.NewStdioWrapper(term) - globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr() - - return runBackup(ctx, backupOptions, globalOptions, term, args) + term, cancel := setupTermstatus(cmd.Context()) + defer cancel() + return runBackup(cmd.Context(), backupOptions, globalOptions, term, args) }, } diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 6045a5d41..b5c62fdea 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -3,7 +3,6 @@ package main import ( "context" "strings" - "sync" "time" "github.com/restic/restic/internal/debug" @@ -38,31 +37,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() - var wg sync.WaitGroup - cancelCtx, cancel := context.WithCancel(ctx) - defer func() { - // shutdown termstatus - cancel() - wg.Wait() - }() - - term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet) - wg.Add(1) - go func() { - defer wg.Done() - term.Run(cancelCtx) - }() - - // allow usage of warnf / verbosef - prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr - defer func() { - globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr - }() - stdioWrapper := ui.NewStdioWrapper(term) - globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr() - - return runRestore(ctx, restoreOptions, globalOptions, term, args) + term, cancel := setupTermstatus(cmd.Context()) + defer cancel() + return runRestore(cmd.Context(), restoreOptions, globalOptions, term, args) }, } diff --git a/cmd/restic/termstatus.go b/cmd/restic/termstatus.go new file mode 100644 index 000000000..e39054427 --- /dev/null +++ b/cmd/restic/termstatus.go @@ -0,0 +1,42 @@ +package main + +import ( + "context" + "sync" + + "github.com/restic/restic/internal/ui" + "github.com/restic/restic/internal/ui/termstatus" +) + +// setupTermstatus creates a new termstatus and reroutes globalOptions.{stdout,stderr} to it +// The returned function must be called to shut down the termstatus, +// +// Expected usage: +// ``` +// term, cancel := setupTermstatus(ctx) +// defer cancel() +// // do stuff +// ``` +func setupTermstatus(ctx context.Context) (*termstatus.Terminal, func()) { + var wg sync.WaitGroup + cancelCtx, cancel := context.WithCancel(ctx) + + term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet) + wg.Add(1) + go func() { + defer wg.Done() + term.Run(cancelCtx) + }() + + // use the termstatus for stdout/stderr + prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr + stdioWrapper := ui.NewStdioWrapper(term) + globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr() + + return term, func() { + // shutdown termstatus + globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr + cancel() + wg.Wait() + } +} From 6b65a495b1ca9b3ce7fbad9074b4084eb19c9393 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 18:12:36 +0100 Subject: [PATCH 057/158] backup/restore: fix termstatus initialization The termstatus must only be canceled once the command has returned. Otherwise output may be lost when the context gets canceled. --- cmd/restic/cmd_backup.go | 2 +- cmd/restic/cmd_restore.go | 2 +- cmd/restic/termstatus.go | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 0596ee918..2ea2a4ec5 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -54,7 +54,7 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea }, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - term, cancel := setupTermstatus(cmd.Context()) + term, cancel := setupTermstatus() defer cancel() return runBackup(cmd.Context(), backupOptions, globalOptions, term, args) }, diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index b5c62fdea..1208d30eb 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -37,7 +37,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - term, cancel := setupTermstatus(cmd.Context()) + term, cancel := setupTermstatus() defer cancel() return runRestore(cmd.Context(), restoreOptions, globalOptions, term, args) }, diff --git a/cmd/restic/termstatus.go b/cmd/restic/termstatus.go index e39054427..cf3cd82ee 100644 --- a/cmd/restic/termstatus.go +++ b/cmd/restic/termstatus.go @@ -13,13 +13,14 @@ import ( // // Expected usage: // ``` -// term, cancel := setupTermstatus(ctx) +// term, cancel := setupTermstatus() // defer cancel() // // do stuff // ``` -func setupTermstatus(ctx context.Context) (*termstatus.Terminal, func()) { +func setupTermstatus() (*termstatus.Terminal, func()) { var wg sync.WaitGroup - cancelCtx, cancel := context.WithCancel(ctx) + // only shutdown once cancel is called to ensure that no output is lost + cancelCtx, cancel := context.WithCancel(context.Background()) term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet) wg.Add(1) From d7a50fe739e3e252ab8089b7b369aa889142ca4c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 18:34:11 +0100 Subject: [PATCH 058/158] properly show termstatus progress bar if visible less than one frame If a progress bar using termstatus was only visible for less than one frame, then its output could be lost. --- cmd/restic/progress.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/cmd/restic/progress.go b/cmd/restic/progress.go index 8b33f94c9..9d93863ad 100644 --- a/cmd/restic/progress.go +++ b/cmd/restic/progress.go @@ -30,7 +30,7 @@ func calculateProgressInterval(show bool, json bool) time.Duration { } // newTerminalProgressMax returns a progress.Counter that prints to stdout or terminal if provided. -func newGenericProgressMax(show bool, max uint64, description string, print func(status string)) *progress.Counter { +func newGenericProgressMax(show bool, max uint64, description string, print func(status string, final bool)) *progress.Counter { if !show { return nil } @@ -46,16 +46,18 @@ func newGenericProgressMax(show bool, max uint64, description string, print func ui.FormatDuration(d), ui.FormatPercent(v, max), v, max, description) } - print(status) - if final { - fmt.Print("\n") - } + print(status, final) }) } func newTerminalProgressMax(show bool, max uint64, description string, term *termstatus.Terminal) *progress.Counter { - return newGenericProgressMax(show, max, description, func(status string) { - term.SetStatus([]string{status}) + return newGenericProgressMax(show, max, description, func(status string, final bool) { + if final { + term.SetStatus([]string{}) + term.Print(status) + } else { + term.SetStatus([]string{status}) + } }) } @@ -64,7 +66,7 @@ func newProgressMax(show bool, max uint64, description string) *progress.Counter return newGenericProgressMax(show, max, description, printProgress) } -func printProgress(status string) { +func printProgress(status string, final bool) { canUpdateStatus := stdoutCanUpdateStatus() @@ -95,6 +97,9 @@ func printProgress(status string) { } _, _ = os.Stdout.Write([]byte(clear + status + carriageControl)) + if final { + _, _ = os.Stdout.Write([]byte("\n")) + } } func newIndexProgress(quiet bool, json bool) *progress.Counter { From feeab84204fab8964793cc13c9264cad255ded80 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 18:40:22 +0100 Subject: [PATCH 059/158] repair pack: extract the repair logic into the repository package Currently, the cmd/restic package contains a significant amount of code that modifies repository internals. This code should in the mid-term move into the repository package. --- cmd/restic/cmd_repair_packs.go | 75 ++++--------------------- cmd/restic/progress.go | 18 ++++++ internal/repository/repair_pack.go | 88 ++++++++++++++++++++++++++++++ internal/ui/progress/printer.go | 30 ++++++++++ 4 files changed, 148 insertions(+), 63 deletions(-) create mode 100644 internal/repository/repair_pack.go create mode 100644 internal/ui/progress/printer.go diff --git a/cmd/restic/cmd_repair_packs.go b/cmd/restic/cmd_repair_packs.go index c572e02c5..04b06c33b 100644 --- a/cmd/restic/cmd_repair_packs.go +++ b/cmd/restic/cmd_repair_packs.go @@ -9,8 +9,8 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/termstatus" "github.com/spf13/cobra" - "golang.org/x/sync/errgroup" ) var cmdRepairPacks = &cobra.Command{ @@ -29,7 +29,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, args []string) error { - return runRepairPacks(cmd.Context(), globalOptions, args) + term, cancel := setupTermstatus() + defer cancel() + return runRepairPacks(cmd.Context(), globalOptions, term, args) }, } @@ -37,7 +39,7 @@ func init() { cmdRepair.AddCommand(cmdRepairPacks) } -func runRepairPacks(ctx context.Context, gopts GlobalOptions, args []string) error { +func runRepairPacks(ctx context.Context, gopts GlobalOptions, term *termstatus.Terminal, args []string) error { // FIXME discuss and add proper feature flag mechanism flag, _ := os.LookupEnv("RESTIC_FEATURES") if flag != "repair-packs-v1" { @@ -68,21 +70,19 @@ func runRepairPacks(ctx context.Context, gopts GlobalOptions, args []string) err return err } - return repairPacks(ctx, gopts, repo, ids) -} - -func repairPacks(ctx context.Context, gopts GlobalOptions, repo *repository.Repository, ids restic.IDSet) error { bar := newIndexProgress(gopts.Quiet, gopts.JSON) - err := repo.LoadIndex(ctx, bar) + err = repo.LoadIndex(ctx, bar) if err != nil { return errors.Fatalf("%s", err) } - Warnf("saving backup copies of pack files in current folder\n") + printer := newTerminalProgressPrinter(gopts.verbosity, term) + + printer.P("saving backup copies of pack files to current folder") for id := range ids { f, err := os.OpenFile("pack-"+id.String(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o666) if err != nil { - return errors.Fatalf("%s", err) + return err } err = repo.Backend().Load(ctx, backend.Handle{Type: restic.PackFile, Name: id.String()}, 0, 0, func(rd io.Reader) error { @@ -94,66 +94,15 @@ func repairPacks(ctx context.Context, gopts GlobalOptions, repo *repository.Repo return err }) if err != nil { - return errors.Fatalf("%s", err) + return err } } - wg, wgCtx := errgroup.WithContext(ctx) - repo.StartPackUploader(wgCtx, wg) - repo.DisableAutoIndexUpdate() - - Warnf("salvaging intact data from specified pack files\n") - bar = newProgressMax(!gopts.Quiet, uint64(len(ids)), "pack files") - defer bar.Done() - - wg.Go(func() error { - // examine all data the indexes have for the pack file - for b := range repo.Index().ListPacks(wgCtx, ids) { - blobs := b.Blobs - if len(blobs) == 0 { - Warnf("no blobs found for pack %v\n", b.PackID) - bar.Add(1) - continue - } - - err = repo.LoadBlobsFromPack(wgCtx, b.PackID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { - if err != nil { - // Fallback path - buf, err = repo.LoadBlob(wgCtx, blob.Type, blob.ID, nil) - if err != nil { - Warnf("failed to load blob %v: %v\n", blob.ID, err) - return nil - } - } - id, _, _, err := repo.SaveBlob(wgCtx, blob.Type, buf, restic.ID{}, true) - if !id.Equal(blob.ID) { - panic("pack id mismatch during upload") - } - return err - }) - if err != nil { - return err - } - bar.Add(1) - } - return repo.Flush(wgCtx) - }) - - if err := wg.Wait(); err != nil { - return errors.Fatalf("%s", err) - } - bar.Done() - - // remove salvaged packs from index - err = rebuildIndexFiles(ctx, gopts, repo, ids, nil, false) + err = repository.RepairPacks(ctx, repo, ids, printer) if err != nil { return errors.Fatalf("%s", err) } - // cleanup - Warnf("removing salvaged pack files\n") - DeleteFiles(ctx, gopts, repo, ids, restic.PackFile) - Warnf("\nUse `restic repair snapshots --forget` to remove the corrupted data blobs from all snapshots\n") return nil } diff --git a/cmd/restic/progress.go b/cmd/restic/progress.go index 9d93863ad..48aa209a6 100644 --- a/cmd/restic/progress.go +++ b/cmd/restic/progress.go @@ -109,3 +109,21 @@ func newIndexProgress(quiet bool, json bool) *progress.Counter { func newIndexTerminalProgress(quiet bool, json bool, term *termstatus.Terminal) *progress.Counter { return newTerminalProgressMax(!quiet && !json && stdoutIsTerminal(), 0, "index files loaded", term) } + +type terminalProgressPrinter struct { + term *termstatus.Terminal + ui.Message + show bool +} + +func (t *terminalProgressPrinter) NewCounter(description string) *progress.Counter { + return newTerminalProgressMax(t.show, 0, description, t.term) +} + +func newTerminalProgressPrinter(verbosity uint, term *termstatus.Terminal) progress.Printer { + return &terminalProgressPrinter{ + term: term, + Message: *ui.NewMessage(term, verbosity), + show: verbosity > 0, + } +} diff --git a/internal/repository/repair_pack.go b/internal/repository/repair_pack.go new file mode 100644 index 000000000..5f3d43dc3 --- /dev/null +++ b/internal/repository/repair_pack.go @@ -0,0 +1,88 @@ +package repository + +import ( + "context" + "errors" + "io" + + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/progress" + "golang.org/x/sync/errgroup" +) + +func RepairPacks(ctx context.Context, repo *Repository, ids restic.IDSet, printer progress.Printer) error { + wg, wgCtx := errgroup.WithContext(ctx) + repo.StartPackUploader(wgCtx, wg) + repo.DisableAutoIndexUpdate() + + printer.P("salvaging intact data from specified pack files") + bar := printer.NewCounter("pack files") + bar.SetMax(uint64(len(ids))) + defer bar.Done() + + wg.Go(func() error { + // examine all data the indexes have for the pack file + for b := range repo.Index().ListPacks(wgCtx, ids) { + blobs := b.Blobs + if len(blobs) == 0 { + printer.E("no blobs found for pack %v", b.PackID) + bar.Add(1) + continue + } + + err := repo.LoadBlobsFromPack(wgCtx, b.PackID, blobs, func(blob restic.BlobHandle, buf []byte, err error) error { + if err != nil { + // Fallback path + buf, err = repo.LoadBlob(wgCtx, blob.Type, blob.ID, nil) + if err != nil { + printer.E("failed to load blob %v: %v", blob.ID, err) + return nil + } + } + id, _, _, err := repo.SaveBlob(wgCtx, blob.Type, buf, restic.ID{}, true) + if !id.Equal(blob.ID) { + panic("pack id mismatch during upload") + } + return err + }) + if err != nil { + return err + } + bar.Add(1) + } + return repo.Flush(wgCtx) + }) + + err := wg.Wait() + bar.Done() + if err != nil { + return err + } + + // remove salvaged packs from index + printer.P("rebuilding index") + + bar = printer.NewCounter("packs processed") + err = repo.Index().Save(ctx, repo, ids, nil, restic.MasterIndexSaveOpts{ + SaveProgress: bar, + DeleteProgress: func() *progress.Counter { + return printer.NewCounter("old indexes deleted") + }, + DeleteReport: func(id restic.ID, err error) { + printer.VV("removed index %v", id.String()) + }, + }) + + if err != nil { + return err + } + + // cleanup + printer.P("removing salvaged pack files") + // if we fail to delete the damaged pack files, then prune will remove them later on + bar = printer.NewCounter("files deleted") + _ = restic.ParallelRemove(ctx, repo, ids, restic.PackFile, nil, bar) + bar.Done() + + return nil +} diff --git a/internal/ui/progress/printer.go b/internal/ui/progress/printer.go new file mode 100644 index 000000000..c95383d3e --- /dev/null +++ b/internal/ui/progress/printer.go @@ -0,0 +1,30 @@ +package progress + +// A Printer can can return a new counter or print messages +// at different log levels. +// It must be safe to call its methods from concurrent goroutines. +type Printer interface { + NewCounter(description string) *Counter + + E(msg string, args ...interface{}) + P(msg string, args ...interface{}) + V(msg string, args ...interface{}) + VV(msg string, args ...interface{}) +} + +// NoopPrinter discards all messages +type NoopPrinter struct{} + +var _ Printer = (*NoopPrinter)(nil) + +func (*NoopPrinter) NewCounter(description string) *Counter { + return nil +} + +func (*NoopPrinter) E(msg string, args ...interface{}) {} + +func (*NoopPrinter) P(msg string, args ...interface{}) {} + +func (*NoopPrinter) V(msg string, args ...interface{}) {} + +func (*NoopPrinter) VV(msg string, args ...interface{}) {} From 7c351bc53c51337e9fd615f5db082ff6c593d7bd Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 21:52:27 +0100 Subject: [PATCH 060/158] repair pack: reenable auto index updates The method is not available on the restic.Repository interface that is used for testing. Drop the call as a small amount of additional index writes is not a problem. --- internal/repository/repair_pack.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/repository/repair_pack.go b/internal/repository/repair_pack.go index 5f3d43dc3..bedbc31df 100644 --- a/internal/repository/repair_pack.go +++ b/internal/repository/repair_pack.go @@ -10,10 +10,9 @@ import ( "golang.org/x/sync/errgroup" ) -func RepairPacks(ctx context.Context, repo *Repository, ids restic.IDSet, printer progress.Printer) error { +func RepairPacks(ctx context.Context, repo restic.Repository, ids restic.IDSet, printer progress.Printer) error { wg, wgCtx := errgroup.WithContext(ctx) repo.StartPackUploader(wgCtx, wg) - repo.DisableAutoIndexUpdate() printer.P("salvaging intact data from specified pack files") bar := printer.NewCounter("pack files") From 764b0bacd613ea4a84db16c96d6eb24c727d1e55 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 21:54:27 +0100 Subject: [PATCH 061/158] repair pack: add support for truncated files --- internal/repository/repair_pack.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/repository/repair_pack.go b/internal/repository/repair_pack.go index bedbc31df..64279e727 100644 --- a/internal/repository/repair_pack.go +++ b/internal/repository/repair_pack.go @@ -44,7 +44,8 @@ func RepairPacks(ctx context.Context, repo restic.Repository, ids restic.IDSet, } return err }) - if err != nil { + // ignore truncated file parts + if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { return err } bar.Add(1) From 42c9318b9c50957bd7bb68e0d13dfd18bdc3e9e0 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 21:54:47 +0100 Subject: [PATCH 062/158] repair pack: add tests --- internal/repository/repack_test.go | 3 +- internal/repository/repair_pack_test.go | 130 ++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 internal/repository/repair_pack_test.go diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index bab04f6b7..3de077a7d 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -62,7 +62,7 @@ func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData fl } } -func createRandomWrongBlob(t testing.TB, repo restic.Repository) { +func createRandomWrongBlob(t testing.TB, repo restic.Repository) restic.BlobHandle { length := randomSize(10*1024, 1024*1024) // 10KiB to 1MiB of data buf := make([]byte, length) rand.Read(buf) @@ -80,6 +80,7 @@ func createRandomWrongBlob(t testing.TB, repo restic.Repository) { if err := repo.Flush(context.Background()); err != nil { t.Fatalf("repo.Flush() returned error %v", err) } + return restic.BlobHandle{ID: id, Type: restic.DataBlob} } // selectBlobs splits the list of all blobs randomly into two lists. A blob diff --git a/internal/repository/repair_pack_test.go b/internal/repository/repair_pack_test.go new file mode 100644 index 000000000..e37c42eb7 --- /dev/null +++ b/internal/repository/repair_pack_test.go @@ -0,0 +1,130 @@ +package repository_test + +import ( + "context" + "math/rand" + "testing" + "time" + + "github.com/restic/restic/internal/backend" + "github.com/restic/restic/internal/index" + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" + rtest "github.com/restic/restic/internal/test" + "github.com/restic/restic/internal/ui/progress" +) + +func listBlobs(repo restic.Repository) restic.BlobSet { + blobs := restic.NewBlobSet() + repo.Index().Each(context.TODO(), func(pb restic.PackedBlob) { + blobs.Insert(pb.BlobHandle) + }) + return blobs +} + +func replaceFile(t *testing.T, repo restic.Repository, h backend.Handle, damage func([]byte) []byte) { + buf, err := backend.LoadAll(context.TODO(), nil, repo.Backend(), h) + test.OK(t, err) + buf = damage(buf) + test.OK(t, repo.Backend().Remove(context.TODO(), h)) + test.OK(t, repo.Backend().Save(context.TODO(), h, backend.NewByteReader(buf, repo.Backend().Hasher()))) +} + +func TestRepairBrokenPack(t *testing.T) { + repository.TestAllVersions(t, testRepairBrokenPack) +} + +func testRepairBrokenPack(t *testing.T, version uint) { + tests := []struct { + name string + damage func(repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) + }{ + { + "valid pack", + func(repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { + return packsBefore, restic.NewBlobSet() + }, + }, + { + "broken pack", + func(repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { + wrongBlob := createRandomWrongBlob(t, repo) + damagedPacks := findPacksForBlobs(t, repo, restic.NewBlobSet(wrongBlob)) + return damagedPacks, restic.NewBlobSet(wrongBlob) + }, + }, + { + "partially broken pack", + func(repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { + // damage one of the pack files + damagedID := packsBefore.List()[0] + replaceFile(t, repo, backend.Handle{Type: backend.PackFile, Name: damagedID.String()}, + func(buf []byte) []byte { + buf[0] ^= 0xff + return buf + }) + + // find blob that starts at offset 0 + var damagedBlob restic.BlobHandle + for blobs := range repo.Index().ListPacks(context.TODO(), restic.NewIDSet(damagedID)) { + for _, blob := range blobs.Blobs { + if blob.Offset == 0 { + damagedBlob = blob.BlobHandle + } + } + } + + return restic.NewIDSet(damagedID), restic.NewBlobSet(damagedBlob) + }, + }, { + "truncated pack", + func(repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { + // damage one of the pack files + damagedID := packsBefore.List()[0] + replaceFile(t, repo, backend.Handle{Type: backend.PackFile, Name: damagedID.String()}, + func(buf []byte) []byte { + buf = buf[0:10] + return buf + }) + + // all blobs in the file are broken + damagedBlobs := restic.NewBlobSet() + for blobs := range repo.Index().ListPacks(context.TODO(), restic.NewIDSet(damagedID)) { + for _, blob := range blobs.Blobs { + damagedBlobs.Insert(blob.BlobHandle) + } + } + return restic.NewIDSet(damagedID), damagedBlobs + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + repo := repository.TestRepositoryWithVersion(t, version) + + seed := time.Now().UnixNano() + rand.Seed(seed) + t.Logf("rand seed is %v", seed) + + createRandomBlobs(t, repo, 5, 0.7) + packsBefore := listPacks(t, repo) + blobsBefore := listBlobs(repo) + + toRepair, damagedBlobs := test.damage(repo, packsBefore) + + rtest.OK(t, repository.RepairPacks(context.TODO(), repo, toRepair, &progress.NoopPrinter{})) + // reload index + rtest.OK(t, repo.SetIndex(index.NewMasterIndex())) + repo.LoadIndex(context.TODO(), nil) + + packsAfter := listPacks(t, repo) + blobsAfter := listBlobs(repo) + + rtest.Assert(t, len(packsAfter.Intersect(toRepair)) == 0, "some damaged packs were not removed") + rtest.Assert(t, len(packsBefore.Sub(toRepair).Sub(packsAfter)) == 0, "not-damaged packs were removed") + rtest.Assert(t, blobsBefore.Sub(damagedBlobs).Equals(blobsAfter), "diverging blob lists") + }) + } +} From fd579421dd8c9e1bdd501fe96ea7a926bd432f78 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 21:50:23 +0100 Subject: [PATCH 063/158] repository: deduplicate test --- internal/repository/repository_test.go | 60 +++++++------------------- 1 file changed, 16 insertions(+), 44 deletions(-) diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 272ea94ac..886486136 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -28,10 +28,19 @@ var testSizes = []int{5, 23, 2<<18 + 23, 1 << 20} var rnd = rand.New(rand.NewSource(time.Now().UnixNano())) func TestSave(t *testing.T) { - repository.TestAllVersions(t, testSave) + repository.TestAllVersions(t, testSavePassID) + repository.TestAllVersions(t, testSaveCalculateID) } -func testSave(t *testing.T, version uint) { +func testSavePassID(t *testing.T, version uint) { + testSave(t, version, false) +} + +func testSaveCalculateID(t *testing.T, version uint) { + testSave(t, version, true) +} + +func testSave(t *testing.T, version uint, calculateID bool) { repo := repository.TestRepositoryWithVersion(t, version) for _, size := range testSizes { @@ -45,51 +54,14 @@ func testSave(t *testing.T, version uint) { repo.StartPackUploader(context.TODO(), &wg) // save - sid, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, restic.ID{}, false) + inputID := restic.ID{} + if !calculateID { + inputID = id + } + sid, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, inputID, false) rtest.OK(t, err) - rtest.Equals(t, id, sid) - rtest.OK(t, repo.Flush(context.Background())) - // rtest.OK(t, repo.SaveIndex()) - - // read back - buf, err := repo.LoadBlob(context.TODO(), restic.DataBlob, id, nil) - rtest.OK(t, err) - rtest.Equals(t, size, len(buf)) - - rtest.Assert(t, len(buf) == len(data), - "number of bytes read back does not match: expected %d, got %d", - len(data), len(buf)) - - rtest.Assert(t, bytes.Equal(buf, data), - "data does not match: expected %02x, got %02x", - data, buf) - } -} - -func TestSaveFrom(t *testing.T) { - repository.TestAllVersions(t, testSaveFrom) -} - -func testSaveFrom(t *testing.T, version uint) { - repo := repository.TestRepositoryWithVersion(t, version) - - for _, size := range testSizes { - data := make([]byte, size) - _, err := io.ReadFull(rnd, data) - rtest.OK(t, err) - - id := restic.Hash(data) - - var wg errgroup.Group - repo.StartPackUploader(context.TODO(), &wg) - - // save - id2, _, _, err := repo.SaveBlob(context.TODO(), restic.DataBlob, data, id, false) - rtest.OK(t, err) - rtest.Equals(t, id, id2) - rtest.OK(t, repo.Flush(context.Background())) // read back From f0e1ad2285a455ccb5503ad0363f2ca8ea0fc299 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 21:58:28 +0100 Subject: [PATCH 064/158] fix linter warning --- internal/repository/repair_pack_test.go | 2 +- internal/ui/progress/printer.go | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/repository/repair_pack_test.go b/internal/repository/repair_pack_test.go index e37c42eb7..6b20dbffb 100644 --- a/internal/repository/repair_pack_test.go +++ b/internal/repository/repair_pack_test.go @@ -117,7 +117,7 @@ func testRepairBrokenPack(t *testing.T, version uint) { rtest.OK(t, repository.RepairPacks(context.TODO(), repo, toRepair, &progress.NoopPrinter{})) // reload index rtest.OK(t, repo.SetIndex(index.NewMasterIndex())) - repo.LoadIndex(context.TODO(), nil) + rtest.OK(t, repo.LoadIndex(context.TODO(), nil)) packsAfter := listPacks(t, repo) blobsAfter := listBlobs(repo) diff --git a/internal/ui/progress/printer.go b/internal/ui/progress/printer.go index c95383d3e..a671621e9 100644 --- a/internal/ui/progress/printer.go +++ b/internal/ui/progress/printer.go @@ -17,14 +17,14 @@ type NoopPrinter struct{} var _ Printer = (*NoopPrinter)(nil) -func (*NoopPrinter) NewCounter(description string) *Counter { +func (*NoopPrinter) NewCounter(_ string) *Counter { return nil } -func (*NoopPrinter) E(msg string, args ...interface{}) {} +func (*NoopPrinter) E(_ string, _ ...interface{}) {} -func (*NoopPrinter) P(msg string, args ...interface{}) {} +func (*NoopPrinter) P(_ string, _ ...interface{}) {} -func (*NoopPrinter) V(msg string, args ...interface{}) {} +func (*NoopPrinter) V(_ string, _ ...interface{}) {} -func (*NoopPrinter) VV(msg string, args ...interface{}) {} +func (*NoopPrinter) VV(_ string, _ ...interface{}) {} From bfb56b78e1342d4b5069ad9e175a27c743030e9c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 19 Jan 2024 22:44:50 +0100 Subject: [PATCH 065/158] replace some usages of restic.Repository with more specific interface This should eventually make it easier to test the code. --- cmd/restic/cmd_backup.go | 2 +- cmd/restic/cmd_debug.go | 2 +- cmd/restic/cmd_diff.go | 6 +++--- cmd/restic/cmd_dump.go | 2 +- cmd/restic/cmd_recover.go | 2 +- cmd/restic/cmd_stats.go | 6 +++--- internal/archiver/testing.go | 4 ++-- internal/checker/checker.go | 2 +- internal/dump/common.go | 8 ++++---- internal/fuse/dir.go | 2 +- internal/fuse/fuse_test.go | 6 +++--- internal/index/index_parallel.go | 2 +- internal/repository/repack_test.go | 2 +- internal/repository/repository_test.go | 2 +- internal/restic/lock.go | 4 ++-- internal/restic/lock_test.go | 4 ++-- internal/restic/node.go | 6 +++--- internal/restic/repository.go | 5 +++++ internal/restorer/restorer_test.go | 4 ++-- 19 files changed, 38 insertions(+), 33 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 2ea2a4ec5..635179417 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -411,7 +411,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er // parent returns the ID of the parent snapshot. If there is none, nil is // returned. -func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) { +func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) { if opts.Force { return nil, nil } diff --git a/cmd/restic/cmd_debug.go b/cmd/restic/cmd_debug.go index 49efb904b..a87e7a0c5 100644 --- a/cmd/restic/cmd_debug.go +++ b/cmd/restic/cmd_debug.go @@ -137,7 +137,7 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) }) } -func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error { +func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer) error { return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { Printf("index_id: %v\n", id) if err != nil { diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index 7878613e1..06f2be2ae 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -61,7 +61,7 @@ func init() { f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata") } -func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) { +func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.LoaderUnpacked, desc string) (*restic.Snapshot, string, error) { sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc) if err != nil { return nil, "", errors.Fatal(err.Error()) @@ -71,7 +71,7 @@ func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, // Comparer collects all things needed to compare two snapshots. type Comparer struct { - repo restic.Repository + repo restic.BlobLoader opts DiffOptions printChange func(change *Change) } @@ -147,7 +147,7 @@ type DiffStatsContainer struct { } // updateBlobs updates the blob counters in the stats struct. -func updateBlobs(repo restic.Repository, blobs restic.BlobSet, stats *DiffStat) { +func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat) { for h := range blobs { switch h.Type { case restic.DataBlob: diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index e6020d847..e72b51fa2 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -67,7 +67,7 @@ func splitPath(p string) []string { return append(s, f) } -func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, d *dump.Dumper) error { +func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper) error { // If we print / we need to assume that there are multiple nodes at that // level in the tree. if pathComponents[0] == "" { diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index ae6aff740..499abdf80 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -158,7 +158,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error { } -func createSnapshot(ctx context.Context, name, hostname string, tags []string, repo restic.Repository, tree *restic.ID) error { +func createSnapshot(ctx context.Context, name, hostname string, tags []string, repo restic.SaverUnpacked, tree *restic.ID) error { sn, err := restic.NewSnapshot([]string{name}, tags, hostname, time.Now()) if err != nil { return errors.Fatalf("unable to save snapshot: %v", err) diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index b0837510d..87c3f8d22 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -189,7 +189,7 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args return nil } -func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo restic.Repository, opts StatsOptions, stats *statsContainer) error { +func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo restic.Loader, opts StatsOptions, stats *statsContainer) error { if snapshot.Tree == nil { return fmt.Errorf("snapshot %s has nil tree", snapshot.ID().Str()) } @@ -211,7 +211,7 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest return nil } -func statsWalkTree(repo restic.Repository, opts StatsOptions, stats *statsContainer, hardLinkIndex *restorer.HardlinkIndex[struct{}]) walker.WalkFunc { +func statsWalkTree(repo restic.Loader, opts StatsOptions, stats *statsContainer, hardLinkIndex *restorer.HardlinkIndex[struct{}]) walker.WalkFunc { return func(parentTreeID restic.ID, npath string, node *restic.Node, nodeErr error) error { if nodeErr != nil { return nodeErr @@ -363,7 +363,7 @@ func statsDebug(ctx context.Context, repo restic.Repository) error { return nil } -func statsDebugFileType(ctx context.Context, repo restic.Repository, tpe restic.FileType) (*sizeHistogram, error) { +func statsDebugFileType(ctx context.Context, repo restic.Lister, tpe restic.FileType) (*sizeHistogram, error) { hist := newSizeHistogram(2 * repository.MaxPackSize) err := repo.List(ctx, tpe, func(id restic.ID, size int64) error { hist.Add(uint64(size)) diff --git a/internal/archiver/testing.go b/internal/archiver/testing.go index c7482d160..111c1e68c 100644 --- a/internal/archiver/testing.go +++ b/internal/archiver/testing.go @@ -209,7 +209,7 @@ func TestEnsureFiles(t testing.TB, target string, dir TestDir) { } // TestEnsureFileContent checks if the file in the repo is the same as file. -func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.Repository, filename string, node *restic.Node, file TestFile) { +func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.BlobLoader, filename string, node *restic.Node, file TestFile) { if int(node.Size) != len(file.Content) { t.Fatalf("%v: wrong node size: want %d, got %d", filename, node.Size, len(file.Content)) return @@ -237,7 +237,7 @@ func TestEnsureFileContent(ctx context.Context, t testing.TB, repo restic.Reposi // TestEnsureTree checks that the tree ID in the repo matches dir. On Windows, // Symlinks are ignored. -func TestEnsureTree(ctx context.Context, t testing.TB, prefix string, repo restic.Repository, treeID restic.ID, dir TestDir) { +func TestEnsureTree(ctx context.Context, t testing.TB, prefix string, repo restic.BlobLoader, treeID restic.ID, dir TestDir) { t.Helper() tree, err := restic.LoadTree(ctx, repo, treeID) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index e6a7a9035..5d324795d 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -362,7 +362,7 @@ func (c *Checker) checkTreeWorker(ctx context.Context, trees <-chan restic.TreeI } } -func loadSnapshotTreeIDs(ctx context.Context, lister restic.Lister, repo restic.Repository) (ids restic.IDs, errs []error) { +func loadSnapshotTreeIDs(ctx context.Context, lister restic.Lister, repo restic.LoaderUnpacked) (ids restic.IDs, errs []error) { err := restic.ForAllSnapshots(ctx, lister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error { if err != nil { errs = append(errs, err) diff --git a/internal/dump/common.go b/internal/dump/common.go index 3ca1ced82..7a8d95ae9 100644 --- a/internal/dump/common.go +++ b/internal/dump/common.go @@ -16,11 +16,11 @@ import ( type Dumper struct { cache *bloblru.Cache format string - repo restic.Repository + repo restic.BlobLoader w io.Writer } -func New(format string, repo restic.Repository, w io.Writer) *Dumper { +func New(format string, repo restic.BlobLoader, w io.Writer) *Dumper { return &Dumper{ cache: bloblru.New(64 << 20), format: format, @@ -47,7 +47,7 @@ func (d *Dumper) DumpTree(ctx context.Context, tree *restic.Tree, rootPath strin } } -func sendTrees(ctx context.Context, repo restic.Repository, tree *restic.Tree, rootPath string, ch chan *restic.Node) { +func sendTrees(ctx context.Context, repo restic.BlobLoader, tree *restic.Tree, rootPath string, ch chan *restic.Node) { defer close(ch) for _, root := range tree.Nodes { @@ -58,7 +58,7 @@ func sendTrees(ctx context.Context, repo restic.Repository, tree *restic.Tree, r } } -func sendNodes(ctx context.Context, repo restic.Repository, root *restic.Node, ch chan *restic.Node) error { +func sendNodes(ctx context.Context, repo restic.BlobLoader, root *restic.Node, ch chan *restic.Node) error { select { case ch <- root: case <-ctx.Done(): diff --git a/internal/fuse/dir.go b/internal/fuse/dir.go index c5aaf6f52..763a9640c 100644 --- a/internal/fuse/dir.go +++ b/internal/fuse/dir.go @@ -58,7 +58,7 @@ func unwrapCtxCanceled(err error) error { // replaceSpecialNodes replaces nodes with name "." and "/" by their contents. // Otherwise, the node is returned. -func replaceSpecialNodes(ctx context.Context, repo restic.Repository, node *restic.Node) ([]*restic.Node, error) { +func replaceSpecialNodes(ctx context.Context, repo restic.BlobLoader, node *restic.Node) ([]*restic.Node, error) { if node.Type != "dir" || node.Subtree == nil { return []*restic.Node{node}, nil } diff --git a/internal/fuse/fuse_test.go b/internal/fuse/fuse_test.go index 0a121b986..1053d49a4 100644 --- a/internal/fuse/fuse_test.go +++ b/internal/fuse/fuse_test.go @@ -37,7 +37,7 @@ func testRead(t testing.TB, f fs.Handle, offset, length int, data []byte) { rtest.OK(t, fr.Read(ctx, req, resp)) } -func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) { +func firstSnapshotID(t testing.TB, repo restic.Lister) (first restic.ID) { err := repo.List(context.TODO(), restic.SnapshotFile, func(id restic.ID, size int64) error { if first.IsNull() { first = id @@ -52,14 +52,14 @@ func firstSnapshotID(t testing.TB, repo restic.Repository) (first restic.ID) { return first } -func loadFirstSnapshot(t testing.TB, repo restic.Repository) *restic.Snapshot { +func loadFirstSnapshot(t testing.TB, repo restic.ListerLoaderUnpacked) *restic.Snapshot { id := firstSnapshotID(t, repo) sn, err := restic.LoadSnapshot(context.TODO(), repo, id) rtest.OK(t, err) return sn } -func loadTree(t testing.TB, repo restic.Repository, id restic.ID) *restic.Tree { +func loadTree(t testing.TB, repo restic.Loader, id restic.ID) *restic.Tree { tree, err := restic.LoadTree(context.TODO(), repo, id) rtest.OK(t, err) return tree diff --git a/internal/index/index_parallel.go b/internal/index/index_parallel.go index d505d756e..3c16d049b 100644 --- a/internal/index/index_parallel.go +++ b/internal/index/index_parallel.go @@ -11,7 +11,7 @@ import ( // ForAllIndexes loads all index files in parallel and calls the given callback. // It is guaranteed that the function is not run concurrently. If the callback // returns an error, this function is cancelled and also returns that error. -func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.Repository, +func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.ListerLoaderUnpacked, fn func(id restic.ID, index *Index, oldFormat bool, err error) error) error { // decoding an index can take quite some time such that this can be both CPU- or IO-bound diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index 3de077a7d..63845d5b1 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -120,7 +120,7 @@ func selectBlobs(t *testing.T, repo restic.Repository, p float32) (list1, list2 return list1, list2 } -func listPacks(t *testing.T, repo restic.Repository) restic.IDSet { +func listPacks(t *testing.T, repo restic.Lister) restic.IDSet { list := restic.NewIDSet() err := repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error { list.Insert(id) diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index 886486136..0fa8e4d4a 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -229,7 +229,7 @@ func TestRepositoryLoadIndex(t *testing.T) { } // loadIndex loads the index id from backend and returns it. -func loadIndex(ctx context.Context, repo restic.Repository, id restic.ID) (*index.Index, error) { +func loadIndex(ctx context.Context, repo restic.LoaderUnpacked, id restic.ID) (*index.Index, error) { buf, err := repo.LoadUnpacked(ctx, restic.IndexFile, id) if err != nil { return nil, err diff --git a/internal/restic/lock.go b/internal/restic/lock.go index 2d7e04f87..8c1001b48 100644 --- a/internal/restic/lock.go +++ b/internal/restic/lock.go @@ -379,7 +379,7 @@ func init() { } // LoadLock loads and unserializes a lock from a repository. -func LoadLock(ctx context.Context, repo Repository, id ID) (*Lock, error) { +func LoadLock(ctx context.Context, repo LoaderUnpacked, id ID) (*Lock, error) { lock := &Lock{} if err := LoadJSONUnpacked(ctx, repo, LockFile, id, lock); err != nil { return nil, err @@ -429,7 +429,7 @@ func RemoveAllLocks(ctx context.Context, repo Repository) (uint, error) { // It is guaranteed that the function is not run concurrently. If the // callback returns an error, this function is cancelled and also returns that error. // If a lock ID is passed via excludeID, it will be ignored. -func ForAllLocks(ctx context.Context, repo Repository, excludeIDs IDSet, fn func(ID, *Lock, error) error) error { +func ForAllLocks(ctx context.Context, repo ListerLoaderUnpacked, excludeIDs IDSet, fn func(ID, *Lock, error) error) error { var m sync.Mutex // For locks decoding is nearly for free, thus just assume were only limited by IO diff --git a/internal/restic/lock_test.go b/internal/restic/lock_test.go index faf3f3593..13b66a432 100644 --- a/internal/restic/lock_test.go +++ b/internal/restic/lock_test.go @@ -120,7 +120,7 @@ func TestExclusiveLockOnLockedRepo(t *testing.T) { rtest.OK(t, elock.Unlock()) } -func createFakeLock(repo restic.Repository, t time.Time, pid int) (restic.ID, error) { +func createFakeLock(repo restic.SaverUnpacked, t time.Time, pid int) (restic.ID, error) { hostname, err := os.Hostname() if err != nil { return restic.ID{}, err @@ -254,7 +254,7 @@ func TestRemoveAllLocks(t *testing.T) { 3, processed) } -func checkSingleLock(t *testing.T, repo restic.Repository) restic.ID { +func checkSingleLock(t *testing.T, repo restic.Lister) restic.ID { t.Helper() var lockID *restic.ID err := repo.List(context.TODO(), restic.LockFile, func(id restic.ID, size int64) error { diff --git a/internal/restic/node.go b/internal/restic/node.go index 7edc41ce8..1d5bb51af 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -142,7 +142,7 @@ func (node Node) GetExtendedAttribute(a string) []byte { } // CreateAt creates the node at the given path but does NOT restore node meta data. -func (node *Node) CreateAt(ctx context.Context, path string, repo Repository) error { +func (node *Node) CreateAt(ctx context.Context, path string, repo BlobLoader) error { debug.Log("create node %v at %v", node.Name, path) switch node.Type { @@ -264,7 +264,7 @@ func (node Node) createDirAt(path string) error { return nil } -func (node Node) createFileAt(ctx context.Context, path string, repo Repository) error { +func (node Node) createFileAt(ctx context.Context, path string, repo BlobLoader) error { f, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) if err != nil { return errors.WithStack(err) @@ -284,7 +284,7 @@ func (node Node) createFileAt(ctx context.Context, path string, repo Repository) return nil } -func (node Node) writeNodeContent(ctx context.Context, repo Repository, f *os.File) error { +func (node Node) writeNodeContent(ctx context.Context, repo BlobLoader, f *os.File) error { var buf []byte for _, id := range node.Content { buf, err := repo.LoadBlob(ctx, DataBlob, id, buf) diff --git a/internal/restic/repository.go b/internal/restic/repository.go index 1c6c8d39d..66cc22ea9 100644 --- a/internal/restic/repository.go +++ b/internal/restic/repository.go @@ -113,3 +113,8 @@ type MasterIndex interface { type Lister interface { List(ctx context.Context, t FileType, fn func(ID, int64) error) error } + +type ListerLoaderUnpacked interface { + Lister + LoaderUnpacked +} diff --git a/internal/restorer/restorer_test.go b/internal/restorer/restorer_test.go index d0e7dad6f..c33214bc3 100644 --- a/internal/restorer/restorer_test.go +++ b/internal/restorer/restorer_test.go @@ -40,7 +40,7 @@ type Dir struct { ModTime time.Time } -func saveFile(t testing.TB, repo restic.Repository, node File) restic.ID { +func saveFile(t testing.TB, repo restic.BlobSaver, node File) restic.ID { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -52,7 +52,7 @@ func saveFile(t testing.TB, repo restic.Repository, node File) restic.ID { return id } -func saveDir(t testing.TB, repo restic.Repository, nodes map[string]Node, inode uint64) restic.ID { +func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode uint64) restic.ID { ctx, cancel := context.WithCancel(context.Background()) defer cancel() From 1dfd854769d593f32dab8b0d1dfa3b907ea73851 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 19 Jan 2024 23:31:46 +0100 Subject: [PATCH 066/158] lock: switch to repository.List --- internal/restic/lock.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/restic/lock.go b/internal/restic/lock.go index 8c1001b48..d98f8ec94 100644 --- a/internal/restic/lock.go +++ b/internal/restic/lock.go @@ -341,8 +341,8 @@ func (l *Lock) checkExistence(ctx context.Context) (bool, error) { exists := false - err := l.repo.Backend().List(ctx, LockFile, func(fi backend.FileInfo) error { - if fi.Name == l.lockID.String() { + err := l.repo.List(ctx, LockFile, func(id ID, size int64) error { + if id.Equal(*l.lockID) { exists = true } return nil From 246559e654c24761483e5914dc37f7ebe008c3f7 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 19 Jan 2024 23:32:17 +0100 Subject: [PATCH 067/158] check: cleanup s3 legacy detection --- internal/checker/checker.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 5d324795d..df865cb41 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -14,7 +14,6 @@ import ( "github.com/minio/sha256-simd" "github.com/restic/restic/internal/backend" "github.com/restic/restic/internal/backend/s3" - "github.com/restic/restic/internal/cache" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/hashing" @@ -241,17 +240,8 @@ func IsOrphanedPack(err error) bool { } func isS3Legacy(b backend.Backend) bool { - // unwrap cache - if be, ok := b.(*cache.Backend); ok { - b = be.Backend - } - - be, ok := b.(*s3.Backend) - if !ok { - return false - } - - return be.Layout.Name() == "s3legacy" + be := backend.AsBackend[*s3.Backend](b) + return be != nil && be.Layout.Name() == "s3legacy" } // Packs checks that all packs referenced in the index are still available and From a9310948cffc56b75cb29bd079ee5794e4548b4d Mon Sep 17 00:00:00 2001 From: Nils Decker Date: Mon, 30 Oct 2023 11:51:11 +0100 Subject: [PATCH 068/158] command ls: add option for ncdu output NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories. It has an option to save a directory tree and analyse it later. This patch adds an output option to the ls command. A snapshot can be seen with `restic ls latest --ncdu | ncdu -f -` - https://dev.yorhel.nl/ncdu --- cmd/restic/cmd_ls.go | 88 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index d30e2819c..755addfe1 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -3,7 +3,10 @@ package main import ( "context" "encoding/json" + "fmt" + "io" "os" + "path/filepath" "strings" "time" @@ -51,6 +54,7 @@ type LsOptions struct { restic.SnapshotFilter Recursive bool HumanReadable bool + Ncdu bool } var lsOptions LsOptions @@ -63,6 +67,7 @@ func init() { flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode") flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories") flags.BoolVar(&lsOptions.HumanReadable, "human-readable", false, "print sizes in human readable format") + flags.BoolVar(&lsOptions.Ncdu, "ncdu", false, "output NCDU save format (pipe into ncdu -f - ") } type lsSnapshot struct { @@ -114,6 +119,81 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { return enc.Encode(n) } +// lsSnapshotNcdu prints a restic snapshot in Ncdu save format. +// It opens the JSON list. Nodes are added with lsNodeNcdu and the list is closed by lsCloseNcdu. +// Format documentation: https://dev.yorhel.nl/ncdu/jsonfmt +func lsSnapshotNcdu(stdout io.Writer, depth *int, sn *restic.Snapshot) { + const NcduMajorVer = 1 + const NcduMinorVer = 2 + + snapshotBytes, err := json.Marshal(sn) + if err != nil { + Warnf("JSON encode failed: %v\n", err) + } + *depth++ + fmt.Fprintf(stdout, "[%d, %d, %s", NcduMajorVer, NcduMinorVer, string(snapshotBytes)) +} + +func lsNodeNcdu(stdout io.Writer, depth *int, currentPath *string, path string, node *restic.Node) { + type NcduNode struct { + Name string `json:"name"` + Asize uint64 `json:"asize"` + Dsize uint64 `json:"dsize"` + Dev uint64 `json:"dev"` + Ino uint64 `json:"ino"` + NLink uint64 `json:"nlink"` + NotReg bool `json:"notreg"` + Uid uint32 `json:"uid"` + Gid uint32 `json:"gid"` + Mode uint16 `json:"mode"` + Mtime int64 `json:"mtime"` + } + + outNode := NcduNode{ + Name: node.Name, + Asize: node.Size, + Dsize: node.Size, + Dev: node.DeviceID, + Ino: node.Inode, + NLink: node.Links, + NotReg: node.Type != "dir" && node.Type != "file", + Uid: node.UID, + Gid: node.GID, + Mode: uint16(node.Mode), + Mtime: node.ModTime.Unix(), + } + + outJson, err := json.Marshal(outNode) + if err != nil { + Warnf("JSON encode failed: %v\n", err) + } + + thisPath := filepath.Dir(path) + for thisPath != *currentPath { + *depth-- + if *depth < 0 { + panic("cannot find suitable parent directory") + } + fmt.Fprintf(stdout, "\n%s]", strings.Repeat(" ", *depth)) + *currentPath = filepath.Dir(*currentPath) + } + + if node.Type == "dir" { + *currentPath = path + *depth++ + fmt.Fprintf(stdout, ", [\n%s%s", strings.Repeat(" ", *depth), string(outJson)) + } else { + fmt.Fprintf(stdout, ",\n%s%s", strings.Repeat(" ", *depth), string(outJson)) + } +} + +func lsCloseNcdu(stdout io.Writer, depth *int) { + for *depth > 0 { + fmt.Fprintf(stdout, "%s]\n", strings.Repeat(" ", *depth)) + *depth-- + } +} + func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error { if len(args) == 0 { return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'") @@ -205,6 +285,14 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri Warnf("JSON encode failed: %v\n", err) } } + } else if opts.Ncdu { + var depth int + var currentPath = "/" + printSnapshot = func(sn *restic.Snapshot) { lsSnapshotNcdu(globalOptions.stdout, &depth, sn) } + printNode = func(path string, node *restic.Node) { + lsNodeNcdu(globalOptions.stdout, &depth, ¤tPath, path, node) + } + defer lsCloseNcdu(globalOptions.stdout, &depth) } else { printSnapshot = func(sn *restic.Snapshot) { Verbosef("%v filtered by %v:\n", sn, dirs) From b2703a40894c8998137fc706712d7804fadb2437 Mon Sep 17 00:00:00 2001 From: Nils Decker Date: Mon, 30 Oct 2023 12:09:33 +0100 Subject: [PATCH 069/158] add changelog for ls --ncdu --- changelog/unreleased/issue-4549 | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 changelog/unreleased/issue-4549 diff --git a/changelog/unreleased/issue-4549 b/changelog/unreleased/issue-4549 new file mode 100644 index 000000000..4829a9881 --- /dev/null +++ b/changelog/unreleased/issue-4549 @@ -0,0 +1,11 @@ +Enhancement: Add `--ncdu` option to `ls` command + +NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories. +It has an option to save a directory tree and analyse it later. +The `ls` command now supports the `--ncdu` option which outputs information +about a snapshot in the NCDU format. + +You can use it as follows: `restic ls latest --ncdu | ncdu -f -` + +https://github.com/restic/restic/issues/4549 +https://github.com/restic/restic/pull/4550 From 9ecbda059cb782d5cde6c2ed136a2633955bdd8e Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 23:36:08 +0100 Subject: [PATCH 070/158] walker: add callback to inform about leaving a directory --- cmd/restic/cmd_find.go | 8 ++++---- cmd/restic/cmd_ls.go | 6 +++++- cmd/restic/cmd_stats.go | 4 +++- internal/dump/common.go | 4 ++-- internal/walker/walker.go | 26 +++++++++++++++++++------- internal/walker/walker_test.go | 2 +- 6 files changed, 34 insertions(+), 16 deletions(-) diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 33fff864f..04e6ae3dd 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -260,7 +260,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error } f.out.newsn = sn - return walker.Walk(ctx, f.repo, *sn.Tree, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error { + return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { debug.Log("Error loading tree %v: %v", parentTreeID, err) @@ -327,7 +327,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error debug.Log(" found match\n") f.out.PrintPattern(nodepath, node) return nil - }) + }}) } func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { @@ -338,7 +338,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { } f.out.newsn = sn - return walker.Walk(ctx, f.repo, *sn.Tree, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error { + return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { debug.Log("Error loading tree %v: %v", parentTreeID, err) @@ -388,7 +388,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error { } return nil - }) + }}) } var errAllPacksFound = errors.New("all packs found") diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 755addfe1..e38985a26 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -318,7 +318,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri printSnapshot(sn) - err = walker.Walk(ctx, repo, *sn.Tree, func(_ restic.ID, nodepath string, node *restic.Node, err error) error { + processNode := func(_ restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { return err } @@ -349,6 +349,10 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return walker.ErrSkipNode } return nil + } + + err = walker.Walk(ctx, repo, *sn.Tree, walker.WalkVisitor{ + ProcessNode: processNode, }) if err != nil { diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index b0837510d..f7febf4d0 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -203,7 +203,9 @@ func statsWalkSnapshot(ctx context.Context, snapshot *restic.Snapshot, repo rest } hardLinkIndex := restorer.NewHardlinkIndex[struct{}]() - err := walker.Walk(ctx, repo, *snapshot.Tree, statsWalkTree(repo, opts, stats, hardLinkIndex)) + err := walker.Walk(ctx, repo, *snapshot.Tree, walker.WalkVisitor{ + ProcessNode: statsWalkTree(repo, opts, stats, hardLinkIndex), + }) if err != nil { return fmt.Errorf("walking tree %s: %v", *snapshot.Tree, err) } diff --git a/internal/dump/common.go b/internal/dump/common.go index 3ca1ced82..88b59e689 100644 --- a/internal/dump/common.go +++ b/internal/dump/common.go @@ -70,7 +70,7 @@ func sendNodes(ctx context.Context, repo restic.Repository, root *restic.Node, c return nil } - err := walker.Walk(ctx, repo, *root.Subtree, func(_ restic.ID, nodepath string, node *restic.Node, err error) error { + err := walker.Walk(ctx, repo, *root.Subtree, walker.WalkVisitor{ProcessNode: func(_ restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { return err } @@ -91,7 +91,7 @@ func sendNodes(ctx context.Context, repo restic.Repository, root *restic.Node, c } return nil - }) + }}) return err } diff --git a/internal/walker/walker.go b/internal/walker/walker.go index aba2e39e5..1bcdda16e 100644 --- a/internal/walker/walker.go +++ b/internal/walker/walker.go @@ -23,12 +23,20 @@ var ErrSkipNode = errors.New("skip this node") // tree are skipped. type WalkFunc func(parentTreeID restic.ID, path string, node *restic.Node, nodeErr error) (err error) +type WalkVisitor struct { + // If the node is a `dir`, it will be entered afterwards unless `ErrSkipNode` + // was returned. This function is mandatory + ProcessNode WalkFunc + // Optional callback + LeaveDir func(path string) +} + // Walk calls walkFn recursively for each node in root. If walkFn returns an // error, it is passed up the call stack. The trees in ignoreTrees are not // walked. If walkFn ignores trees, these are added to the set. -func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, walkFn WalkFunc) error { +func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, visitor WalkVisitor) error { tree, err := restic.LoadTree(ctx, repo, root) - err = walkFn(root, "/", nil, err) + err = visitor.ProcessNode(root, "/", nil, err) if err != nil { if err == ErrSkipNode { @@ -37,13 +45,13 @@ func Walk(ctx context.Context, repo restic.BlobLoader, root restic.ID, walkFn Wa return err } - return walk(ctx, repo, "/", root, tree, walkFn) + return walk(ctx, repo, "/", root, tree, visitor) } // walk recursively traverses the tree, ignoring subtrees when the ID of the // subtree is in ignoreTrees. If err is nil and ignore is true, the subtree ID // will be added to ignoreTrees by walk. -func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *restic.Tree, walkFn WalkFunc) (err error) { +func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTreeID restic.ID, tree *restic.Tree, visitor WalkVisitor) (err error) { sort.Slice(tree.Nodes, func(i, j int) bool { return tree.Nodes[i].Name < tree.Nodes[j].Name }) @@ -56,7 +64,7 @@ func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTree } if node.Type != "dir" { - err := walkFn(parentTreeID, p, node, nil) + err := visitor.ProcessNode(parentTreeID, p, node, nil) if err != nil { if err == ErrSkipNode { // skip the remaining entries in this tree @@ -74,18 +82,22 @@ func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTree } subtree, err := restic.LoadTree(ctx, repo, *node.Subtree) - err = walkFn(parentTreeID, p, node, err) + err = visitor.ProcessNode(parentTreeID, p, node, err) if err != nil { if err == ErrSkipNode { continue } } - err = walk(ctx, repo, p, *node.Subtree, subtree, walkFn) + err = walk(ctx, repo, p, *node.Subtree, subtree, visitor) if err != nil { return err } } + if visitor.LeaveDir != nil { + visitor.LeaveDir(prefix) + } + return nil } diff --git a/internal/walker/walker_test.go b/internal/walker/walker_test.go index 786570e02..e2d1f866f 100644 --- a/internal/walker/walker_test.go +++ b/internal/walker/walker_test.go @@ -406,7 +406,7 @@ func TestWalker(t *testing.T) { defer cancel() fn, last := check(t) - err := Walk(ctx, repo, root, fn) + err := Walk(ctx, repo, root, WalkVisitor{ProcessNode: fn}) if err != nil { t.Error(err) } From 1b008c92d3292525045df37f8a7ed417731b97a1 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 23:37:54 +0100 Subject: [PATCH 071/158] ls: rework ncdu output to use walker.LeaveDir --- cmd/restic/cmd_ls.go | 72 ++++++++++++++++++++++++-------------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index e38985a26..34b449f2c 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "os" - "path/filepath" "strings" "time" @@ -67,7 +66,7 @@ func init() { flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode") flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories") flags.BoolVar(&lsOptions.HumanReadable, "human-readable", false, "print sizes in human readable format") - flags.BoolVar(&lsOptions.Ncdu, "ncdu", false, "output NCDU save format (pipe into ncdu -f - ") + flags.BoolVar(&lsOptions.Ncdu, "ncdu", false, "output NCDU save format (pipe into 'ncdu -f -')") } type lsSnapshot struct { @@ -119,10 +118,15 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { return enc.Encode(n) } +type ncduPrinter struct { + out io.Writer + depth int +} + // lsSnapshotNcdu prints a restic snapshot in Ncdu save format. // It opens the JSON list. Nodes are added with lsNodeNcdu and the list is closed by lsCloseNcdu. // Format documentation: https://dev.yorhel.nl/ncdu/jsonfmt -func lsSnapshotNcdu(stdout io.Writer, depth *int, sn *restic.Snapshot) { +func (p *ncduPrinter) ProcessSnapshot(sn *restic.Snapshot) { const NcduMajorVer = 1 const NcduMinorVer = 2 @@ -130,11 +134,11 @@ func lsSnapshotNcdu(stdout io.Writer, depth *int, sn *restic.Snapshot) { if err != nil { Warnf("JSON encode failed: %v\n", err) } - *depth++ - fmt.Fprintf(stdout, "[%d, %d, %s", NcduMajorVer, NcduMinorVer, string(snapshotBytes)) + p.depth++ + fmt.Fprintf(p.out, "[%d, %d, %s", NcduMajorVer, NcduMinorVer, string(snapshotBytes)) } -func lsNodeNcdu(stdout io.Writer, depth *int, currentPath *string, path string, node *restic.Node) { +func (p *ncduPrinter) ProcessNode(path string, node *restic.Node) { type NcduNode struct { Name string `json:"name"` Asize uint64 `json:"asize"` @@ -168,30 +172,21 @@ func lsNodeNcdu(stdout io.Writer, depth *int, currentPath *string, path string, Warnf("JSON encode failed: %v\n", err) } - thisPath := filepath.Dir(path) - for thisPath != *currentPath { - *depth-- - if *depth < 0 { - panic("cannot find suitable parent directory") - } - fmt.Fprintf(stdout, "\n%s]", strings.Repeat(" ", *depth)) - *currentPath = filepath.Dir(*currentPath) - } - if node.Type == "dir" { - *currentPath = path - *depth++ - fmt.Fprintf(stdout, ", [\n%s%s", strings.Repeat(" ", *depth), string(outJson)) + p.depth++ + fmt.Fprintf(p.out, ", [\n%s%s", strings.Repeat(" ", p.depth), string(outJson)) } else { - fmt.Fprintf(stdout, ",\n%s%s", strings.Repeat(" ", *depth), string(outJson)) + fmt.Fprintf(p.out, ",\n%s%s", strings.Repeat(" ", p.depth), string(outJson)) } } -func lsCloseNcdu(stdout io.Writer, depth *int) { - for *depth > 0 { - fmt.Fprintf(stdout, "%s]\n", strings.Repeat(" ", *depth)) - *depth-- - } +func (p *ncduPrinter) LeaveDir(path string) { + p.depth-- + fmt.Fprintf(p.out, "\n%s]", strings.Repeat(" ", p.depth)) +} + +func (p *ncduPrinter) Close() { + fmt.Fprint(p.out, "\n]\n") } func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error { @@ -260,8 +255,10 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri } var ( - printSnapshot func(sn *restic.Snapshot) - printNode func(path string, node *restic.Node) + printSnapshot func(sn *restic.Snapshot) + printNode func(path string, node *restic.Node) + printLeaveNode func(path string) + printClose func() ) if gopts.JSON { @@ -286,13 +283,13 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri } } } else if opts.Ncdu { - var depth int - var currentPath = "/" - printSnapshot = func(sn *restic.Snapshot) { lsSnapshotNcdu(globalOptions.stdout, &depth, sn) } - printNode = func(path string, node *restic.Node) { - lsNodeNcdu(globalOptions.stdout, &depth, ¤tPath, path, node) + ncdu := &ncduPrinter{ + out: globalOptions.stdout, } - defer lsCloseNcdu(globalOptions.stdout, &depth) + printSnapshot = ncdu.ProcessSnapshot + printNode = ncdu.ProcessNode + printLeaveNode = ncdu.LeaveDir + printClose = ncdu.Close } else { printSnapshot = func(sn *restic.Snapshot) { Verbosef("%v filtered by %v:\n", sn, dirs) @@ -353,11 +350,20 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri err = walker.Walk(ctx, repo, *sn.Tree, walker.WalkVisitor{ ProcessNode: processNode, + LeaveDir: func(path string) { + if printLeaveNode != nil && withinDir(path) && path != "/" { + printLeaveNode(path) + } + }, }) if err != nil { return err } + if printClose != nil { + printClose() + } + return nil } From a2fe3376104c12f05a3b4c6d703bdaecf057c67a Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 20 Jan 2024 23:59:33 +0100 Subject: [PATCH 072/158] ls: unify printer implementations --- cmd/restic/cmd_ls.go | 126 +++++++++++++++++++++++++------------------ 1 file changed, 73 insertions(+), 53 deletions(-) diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 34b449f2c..127bec4e2 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -69,14 +69,44 @@ func init() { flags.BoolVar(&lsOptions.Ncdu, "ncdu", false, "output NCDU save format (pipe into 'ncdu -f -')") } -type lsSnapshot struct { - *restic.Snapshot - ID *restic.ID `json:"id"` - ShortID string `json:"short_id"` - StructType string `json:"struct_type"` // "snapshot" +type lsPrinter interface { + Snapshot(sn *restic.Snapshot) + Node(path string, node *restic.Node) + LeaveDir(path string) + Close() +} + +type jsonLsPrinter struct { + enc *json.Encoder +} + +func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) { + type lsSnapshot struct { + *restic.Snapshot + ID *restic.ID `json:"id"` + ShortID string `json:"short_id"` + StructType string `json:"struct_type"` // "snapshot" + } + + err := p.enc.Encode(lsSnapshot{ + Snapshot: sn, + ID: sn.ID(), + ShortID: sn.ID().Str(), + StructType: "snapshot", + }) + if err != nil { + Warnf("JSON encode failed: %v\n", err) + } } // Print node in our custom JSON format, followed by a newline. +func (p *jsonLsPrinter) Node(path string, node *restic.Node) { + err := lsNodeJSON(p.enc, path, node) + if err != nil { + Warnf("JSON encode failed: %v\n", err) + } +} + func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { n := &struct { Name string `json:"name"` @@ -118,7 +148,10 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { return enc.Encode(n) } -type ncduPrinter struct { +func (p *jsonLsPrinter) LeaveDir(path string) {} +func (p *jsonLsPrinter) Close() {} + +type ncduLsPrinter struct { out io.Writer depth int } @@ -126,7 +159,7 @@ type ncduPrinter struct { // lsSnapshotNcdu prints a restic snapshot in Ncdu save format. // It opens the JSON list. Nodes are added with lsNodeNcdu and the list is closed by lsCloseNcdu. // Format documentation: https://dev.yorhel.nl/ncdu/jsonfmt -func (p *ncduPrinter) ProcessSnapshot(sn *restic.Snapshot) { +func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) { const NcduMajorVer = 1 const NcduMinorVer = 2 @@ -138,7 +171,7 @@ func (p *ncduPrinter) ProcessSnapshot(sn *restic.Snapshot) { fmt.Fprintf(p.out, "[%d, %d, %s", NcduMajorVer, NcduMinorVer, string(snapshotBytes)) } -func (p *ncduPrinter) ProcessNode(path string, node *restic.Node) { +func (p *ncduLsPrinter) Node(path string, node *restic.Node) { type NcduNode struct { Name string `json:"name"` Asize uint64 `json:"asize"` @@ -180,15 +213,31 @@ func (p *ncduPrinter) ProcessNode(path string, node *restic.Node) { } } -func (p *ncduPrinter) LeaveDir(path string) { +func (p *ncduLsPrinter) LeaveDir(path string) { p.depth-- fmt.Fprintf(p.out, "\n%s]", strings.Repeat(" ", p.depth)) } -func (p *ncduPrinter) Close() { +func (p *ncduLsPrinter) Close() { fmt.Fprint(p.out, "\n]\n") } +type textLsPrinter struct { + dirs []string + ListLong bool + HumanReadable bool +} + +func (p *textLsPrinter) Snapshot(sn *restic.Snapshot) { + Verbosef("%v filtered by %v:\n", sn, p.dirs) +} +func (p *textLsPrinter) Node(path string, node *restic.Node) { + Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable)) +} + +func (p *textLsPrinter) LeaveDir(path string) {} +func (p *textLsPrinter) Close() {} + func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error { if len(args) == 0 { return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'") @@ -254,48 +303,21 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return err } - var ( - printSnapshot func(sn *restic.Snapshot) - printNode func(path string, node *restic.Node) - printLeaveNode func(path string) - printClose func() - ) + var printer lsPrinter if gopts.JSON { - enc := json.NewEncoder(globalOptions.stdout) - - printSnapshot = func(sn *restic.Snapshot) { - err := enc.Encode(lsSnapshot{ - Snapshot: sn, - ID: sn.ID(), - ShortID: sn.ID().Str(), - StructType: "snapshot", - }) - if err != nil { - Warnf("JSON encode failed: %v\n", err) - } - } - - printNode = func(path string, node *restic.Node) { - err := lsNodeJSON(enc, path, node) - if err != nil { - Warnf("JSON encode failed: %v\n", err) - } + printer = &jsonLsPrinter{ + enc: json.NewEncoder(globalOptions.stdout), } } else if opts.Ncdu { - ncdu := &ncduPrinter{ + printer = &ncduLsPrinter{ out: globalOptions.stdout, } - printSnapshot = ncdu.ProcessSnapshot - printNode = ncdu.ProcessNode - printLeaveNode = ncdu.LeaveDir - printClose = ncdu.Close } else { - printSnapshot = func(sn *restic.Snapshot) { - Verbosef("%v filtered by %v:\n", sn, dirs) - } - printNode = func(path string, node *restic.Node) { - Printf("%s\n", formatNode(path, node, opts.ListLong, opts.HumanReadable)) + printer = &textLsPrinter{ + dirs: dirs, + ListLong: opts.ListLong, + HumanReadable: opts.HumanReadable, } } @@ -313,7 +335,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return err } - printSnapshot(sn) + printer.Snapshot(sn) processNode := func(_ restic.ID, nodepath string, node *restic.Node, err error) error { if err != nil { @@ -325,7 +347,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri if withinDir(nodepath) { // if we're within a dir, print the node - printNode(nodepath, node) + printer.Node(nodepath, node) // if recursive listing is requested, signal the walker that it // should continue walking recursively @@ -351,8 +373,9 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri err = walker.Walk(ctx, repo, *sn.Tree, walker.WalkVisitor{ ProcessNode: processNode, LeaveDir: func(path string) { - if printLeaveNode != nil && withinDir(path) && path != "/" { - printLeaveNode(path) + // the root path `/` has no corresponding node and is thus also skipped by processNode + if withinDir(path) && path != "/" { + printer.LeaveDir(path) } }, }) @@ -361,9 +384,6 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri return err } - if printClose != nil { - printClose() - } - + printer.Close() return nil } From 509b339d548b54bb850ef4e6d25031e410bb825b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 21 Jan 2024 15:37:02 +0100 Subject: [PATCH 073/158] ls: correctly handle setuid/setgit/sticky bit in ncdu output --- cmd/restic/cmd_ls.go | 20 ++++- cmd/restic/cmd_ls_test.go | 168 +++++++++++++++++++++++--------------- 2 files changed, 119 insertions(+), 69 deletions(-) diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 127bec4e2..71d3342ff 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -171,7 +171,7 @@ func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) { fmt.Fprintf(p.out, "[%d, %d, %s", NcduMajorVer, NcduMinorVer, string(snapshotBytes)) } -func (p *ncduLsPrinter) Node(path string, node *restic.Node) { +func lsNcduNode(path string, node *restic.Node) ([]byte, error) { type NcduNode struct { Name string `json:"name"` Asize uint64 `json:"asize"` @@ -196,11 +196,25 @@ func (p *ncduLsPrinter) Node(path string, node *restic.Node) { NotReg: node.Type != "dir" && node.Type != "file", Uid: node.UID, Gid: node.GID, - Mode: uint16(node.Mode), + Mode: uint16(node.Mode & os.ModePerm), Mtime: node.ModTime.Unix(), } + // bits according to inode(7) manpage + if node.Mode&os.ModeSetuid != 0 { + outNode.Mode |= 0o4000 + } + if node.Mode&os.ModeSetgid != 0 { + outNode.Mode |= 0o2000 + } + if node.Mode&os.ModeSticky != 0 { + outNode.Mode |= 0o1000 + } - outJson, err := json.Marshal(outNode) + return json.Marshal(outNode) +} + +func (p *ncduLsPrinter) Node(path string, node *restic.Node) { + outJson, err := lsNcduNode(path, node) if err != nil { Warnf("JSON encode failed: %v\n", err) } diff --git a/cmd/restic/cmd_ls_test.go b/cmd/restic/cmd_ls_test.go index 8a4fa51ee..99aa7cf8f 100644 --- a/cmd/restic/cmd_ls_test.go +++ b/cmd/restic/cmd_ls_test.go @@ -11,78 +11,94 @@ import ( rtest "github.com/restic/restic/internal/test" ) +type lsTestNode struct { + path string + restic.Node +} + +var lsTestNodes []lsTestNode = []lsTestNode{ + // Mode is omitted when zero. + // Permissions, by convention is "-" per mode bit + { + path: "/bar/baz", + Node: restic.Node{ + Name: "baz", + Type: "file", + Size: 12345, + UID: 10000000, + GID: 20000000, + + User: "nobody", + Group: "nobodies", + Links: 1, + }, + }, + + // Even empty files get an explicit size. + { + path: "/foo/empty", + Node: restic.Node{ + Name: "empty", + Type: "file", + Size: 0, + UID: 1001, + GID: 1001, + + User: "not printed", + Group: "not printed", + Links: 0xF00, + }, + }, + + // Non-regular files do not get a size. + // Mode is printed in decimal, including the type bits. + { + path: "/foo/link", + Node: restic.Node{ + Name: "link", + Type: "symlink", + Mode: os.ModeSymlink | 0777, + LinkTarget: "not printed", + }, + }, + + { + path: "/some/directory", + Node: restic.Node{ + Name: "directory", + Type: "dir", + Mode: os.ModeDir | 0755, + ModTime: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC), + AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC), + ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC), + }, + }, + + // Test encoding of setuid/setgid/sticky bit + { + path: "/some/sticky", + Node: restic.Node{ + Name: "sticky", + Type: "dir", + Mode: os.ModeDir | 0755 | os.ModeSetuid | os.ModeSetgid | os.ModeSticky, + }, + }, +} + func TestLsNodeJSON(t *testing.T) { - for _, c := range []struct { - path string - restic.Node - expect string - }{ - // Mode is omitted when zero. - // Permissions, by convention is "-" per mode bit - { - path: "/bar/baz", - Node: restic.Node{ - Name: "baz", - Type: "file", - Size: 12345, - UID: 10000000, - GID: 20000000, - - User: "nobody", - Group: "nobodies", - Links: 1, - }, - expect: `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, - }, - - // Even empty files get an explicit size. - { - path: "/foo/empty", - Node: restic.Node{ - Name: "empty", - Type: "file", - Size: 0, - UID: 1001, - GID: 1001, - - User: "not printed", - Group: "not printed", - Links: 0xF00, - }, - expect: `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, - }, - - // Non-regular files do not get a size. - // Mode is printed in decimal, including the type bits. - { - path: "/foo/link", - Node: restic.Node{ - Name: "link", - Type: "symlink", - Mode: os.ModeSymlink | 0777, - LinkTarget: "not printed", - }, - expect: `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, - }, - - { - path: "/some/directory", - Node: restic.Node{ - Name: "directory", - Type: "dir", - Mode: os.ModeDir | 0755, - ModTime: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC), - AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC), - ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC), - }, - expect: `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`, - }, + for i, expect := range []string{ + `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, + `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, + `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, + `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`, + `{"name":"sticky","type":"dir","path":"/some/sticky","uid":0,"gid":0,"mode":2161115629,"permissions":"dugtrwxr-xr-x","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, } { + c := lsTestNodes[i] buf := new(bytes.Buffer) enc := json.NewEncoder(buf) err := lsNodeJSON(enc, c.path, &c.Node) rtest.OK(t, err) - rtest.Equals(t, c.expect+"\n", buf.String()) + rtest.Equals(t, expect+"\n", buf.String()) // Sanity check: output must be valid JSON. var v interface{} @@ -90,3 +106,23 @@ func TestLsNodeJSON(t *testing.T) { rtest.OK(t, err) } } + +func TestLsNcduNode(t *testing.T) { + for i, expect := range []string{ + `{"name":"baz","asize":12345,"dsize":12345,"dev":0,"ino":0,"nlink":1,"notreg":false,"uid":10000000,"gid":20000000,"mode":0,"mtime":-62135596800}`, + `{"name":"empty","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":3840,"notreg":false,"uid":1001,"gid":1001,"mode":0,"mtime":-62135596800}`, + `{"name":"link","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":true,"uid":0,"gid":0,"mode":511,"mtime":-62135596800}`, + `{"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":493,"mtime":1577934245}`, + `{"name":"sticky","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":4077,"mtime":-62135596800}`, + } { + c := lsTestNodes[i] + out, err := lsNcduNode(c.path, &c.Node) + rtest.OK(t, err) + rtest.Equals(t, expect, string(out)) + + // Sanity check: output must be valid JSON. + var v interface{} + err = json.Unmarshal(out, &v) + rtest.OK(t, err) + } +} From 4bae54d04030561cd9fcfcf4986dc78cc6c1d088 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 21 Jan 2024 15:56:07 +0100 Subject: [PATCH 074/158] ls: test ncdu output format --- cmd/restic/cmd_ls.go | 2 +- cmd/restic/cmd_ls_integration_test.go | 36 ++++++++++++++++++++++++--- cmd/restic/cmd_ls_test.go | 31 +++++++++++++++++++++++ 3 files changed, 64 insertions(+), 5 deletions(-) diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 71d3342ff..f8754bfc4 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -220,8 +220,8 @@ func (p *ncduLsPrinter) Node(path string, node *restic.Node) { } if node.Type == "dir" { + fmt.Fprintf(p.out, ",\n%s[\n%s%s", strings.Repeat(" ", p.depth), strings.Repeat(" ", p.depth+1), string(outJson)) p.depth++ - fmt.Fprintf(p.out, ", [\n%s%s", strings.Repeat(" ", p.depth), string(outJson)) } else { fmt.Fprintf(p.out, ",\n%s%s", strings.Repeat(" ", p.depth), string(outJson)) } diff --git a/cmd/restic/cmd_ls_integration_test.go b/cmd/restic/cmd_ls_integration_test.go index 39bf9c3b0..d71d686e1 100644 --- a/cmd/restic/cmd_ls_integration_test.go +++ b/cmd/restic/cmd_ls_integration_test.go @@ -2,18 +2,46 @@ package main import ( "context" + "encoding/json" + "path/filepath" "strings" "testing" rtest "github.com/restic/restic/internal/test" ) -func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string { +func testRunLsWithOpts(t testing.TB, gopts GlobalOptions, opts LsOptions, args []string) []byte { buf, err := withCaptureStdout(func() error { gopts.Quiet = true - opts := LsOptions{} - return runLs(context.TODO(), opts, gopts, []string{snapshotID}) + return runLs(context.TODO(), opts, gopts, args) }) rtest.OK(t, err) - return strings.Split(buf.String(), "\n") + return buf.Bytes() +} + +func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string { + out := testRunLsWithOpts(t, gopts, LsOptions{}, []string{snapshotID}) + return strings.Split(string(out), "\n") +} + +func assertIsValidJson(t *testing.T, data []byte) { + // Sanity check: output must be valid JSON. + var v interface{} + err := json.Unmarshal(data, &v) + rtest.OK(t, err) +} + +func TestRunLsNcdu(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + opts := BackupOptions{} + testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts) + + ncdu := testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest"}) + assertIsValidJson(t, ncdu) + + ncdu = testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest", "/testdata"}) + assertIsValidJson(t, ncdu) } diff --git a/cmd/restic/cmd_ls_test.go b/cmd/restic/cmd_ls_test.go index 99aa7cf8f..34d421144 100644 --- a/cmd/restic/cmd_ls_test.go +++ b/cmd/restic/cmd_ls_test.go @@ -126,3 +126,34 @@ func TestLsNcduNode(t *testing.T) { rtest.OK(t, err) } } + +func TestLsNcdu(t *testing.T) { + var buf bytes.Buffer + printer := &ncduLsPrinter{ + out: &buf, + } + + printer.Snapshot(&restic.Snapshot{ + Hostname: "host", + Paths: []string{"/example"}, + }) + printer.Node("/directory", &restic.Node{ + Type: "dir", + Name: "directory", + }) + printer.Node("/directory/data", &restic.Node{ + Type: "file", + Name: "data", + Size: 42, + }) + printer.LeaveDir("/directory") + printer.Close() + + rtest.Equals(t, `[1, 2, {"time":"0001-01-01T00:00:00Z","tree":null,"paths":["/example"],"hostname":"host"}, + [ + {"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800}, + {"name":"data","asize":42,"dsize":42,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800} + ] +] +`, buf.String()) +} From a2f2f8fb4c22292b15c2b4103cd7e4a6e3a95366 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 21 Jan 2024 15:58:49 +0100 Subject: [PATCH 075/158] fix linter warning --- cmd/restic/cmd_ls.go | 26 +++++++++++++------------- cmd/restic/cmd_ls_integration_test.go | 6 +++--- cmd/restic/cmd_ls_test.go | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index f8754bfc4..3038f98c0 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -148,8 +148,8 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { return enc.Encode(n) } -func (p *jsonLsPrinter) LeaveDir(path string) {} -func (p *jsonLsPrinter) Close() {} +func (p *jsonLsPrinter) LeaveDir(_ string) {} +func (p *jsonLsPrinter) Close() {} type ncduLsPrinter struct { out io.Writer @@ -171,7 +171,7 @@ func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) { fmt.Fprintf(p.out, "[%d, %d, %s", NcduMajorVer, NcduMinorVer, string(snapshotBytes)) } -func lsNcduNode(path string, node *restic.Node) ([]byte, error) { +func lsNcduNode(_ string, node *restic.Node) ([]byte, error) { type NcduNode struct { Name string `json:"name"` Asize uint64 `json:"asize"` @@ -180,8 +180,8 @@ func lsNcduNode(path string, node *restic.Node) ([]byte, error) { Ino uint64 `json:"ino"` NLink uint64 `json:"nlink"` NotReg bool `json:"notreg"` - Uid uint32 `json:"uid"` - Gid uint32 `json:"gid"` + UID uint32 `json:"uid"` + GID uint32 `json:"gid"` Mode uint16 `json:"mode"` Mtime int64 `json:"mtime"` } @@ -194,8 +194,8 @@ func lsNcduNode(path string, node *restic.Node) ([]byte, error) { Ino: node.Inode, NLink: node.Links, NotReg: node.Type != "dir" && node.Type != "file", - Uid: node.UID, - Gid: node.GID, + UID: node.UID, + GID: node.GID, Mode: uint16(node.Mode & os.ModePerm), Mtime: node.ModTime.Unix(), } @@ -214,20 +214,20 @@ func lsNcduNode(path string, node *restic.Node) ([]byte, error) { } func (p *ncduLsPrinter) Node(path string, node *restic.Node) { - outJson, err := lsNcduNode(path, node) + out, err := lsNcduNode(path, node) if err != nil { Warnf("JSON encode failed: %v\n", err) } if node.Type == "dir" { - fmt.Fprintf(p.out, ",\n%s[\n%s%s", strings.Repeat(" ", p.depth), strings.Repeat(" ", p.depth+1), string(outJson)) + fmt.Fprintf(p.out, ",\n%s[\n%s%s", strings.Repeat(" ", p.depth), strings.Repeat(" ", p.depth+1), string(out)) p.depth++ } else { - fmt.Fprintf(p.out, ",\n%s%s", strings.Repeat(" ", p.depth), string(outJson)) + fmt.Fprintf(p.out, ",\n%s%s", strings.Repeat(" ", p.depth), string(out)) } } -func (p *ncduLsPrinter) LeaveDir(path string) { +func (p *ncduLsPrinter) LeaveDir(_ string) { p.depth-- fmt.Fprintf(p.out, "\n%s]", strings.Repeat(" ", p.depth)) } @@ -249,8 +249,8 @@ func (p *textLsPrinter) Node(path string, node *restic.Node) { Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable)) } -func (p *textLsPrinter) LeaveDir(path string) {} -func (p *textLsPrinter) Close() {} +func (p *textLsPrinter) LeaveDir(_ string) {} +func (p *textLsPrinter) Close() {} func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error { if len(args) == 0 { diff --git a/cmd/restic/cmd_ls_integration_test.go b/cmd/restic/cmd_ls_integration_test.go index d71d686e1..1b3c964e4 100644 --- a/cmd/restic/cmd_ls_integration_test.go +++ b/cmd/restic/cmd_ls_integration_test.go @@ -24,7 +24,7 @@ func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string { return strings.Split(string(out), "\n") } -func assertIsValidJson(t *testing.T, data []byte) { +func assertIsValidJSON(t *testing.T, data []byte) { // Sanity check: output must be valid JSON. var v interface{} err := json.Unmarshal(data, &v) @@ -40,8 +40,8 @@ func TestRunLsNcdu(t *testing.T) { testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts) ncdu := testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest"}) - assertIsValidJson(t, ncdu) + assertIsValidJSON(t, ncdu) ncdu = testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest", "/testdata"}) - assertIsValidJson(t, ncdu) + assertIsValidJSON(t, ncdu) } diff --git a/cmd/restic/cmd_ls_test.go b/cmd/restic/cmd_ls_test.go index 34d421144..41c235eab 100644 --- a/cmd/restic/cmd_ls_test.go +++ b/cmd/restic/cmd_ls_test.go @@ -16,7 +16,7 @@ type lsTestNode struct { restic.Node } -var lsTestNodes []lsTestNode = []lsTestNode{ +var lsTestNodes = []lsTestNode{ // Mode is omitted when zero. // Permissions, by convention is "-" per mode bit { From 261737abc89bb8521445372d3c9a429fbd4554d2 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 21 Jan 2024 16:09:05 +0100 Subject: [PATCH 076/158] ls: only allow either --json or --ncdu --- cmd/restic/cmd_ls.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 3038f98c0..f412546ae 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -66,7 +66,7 @@ func init() { flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode") flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories") flags.BoolVar(&lsOptions.HumanReadable, "human-readable", false, "print sizes in human readable format") - flags.BoolVar(&lsOptions.Ncdu, "ncdu", false, "output NCDU save format (pipe into 'ncdu -f -')") + flags.BoolVar(&lsOptions.Ncdu, "ncdu", false, "output NCDU export format (pipe into 'ncdu -f -')") } type lsPrinter interface { @@ -256,6 +256,9 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri if len(args) == 0 { return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'") } + if opts.Ncdu && gopts.JSON { + return errors.Fatal("only either '--json' or '--ncdu' can be specified") + } // extract any specific directories to walk var dirs []string From 2c80cfa4a5c0aa50c353d52a732ac617e17649b9 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 21 Jan 2024 16:43:32 +0100 Subject: [PATCH 077/158] walker: fix missing leaveDir if directory is partially skipped --- internal/walker/walker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/walker/walker.go b/internal/walker/walker.go index 1bcdda16e..091b05489 100644 --- a/internal/walker/walker.go +++ b/internal/walker/walker.go @@ -68,7 +68,7 @@ func walk(ctx context.Context, repo restic.BlobLoader, prefix string, parentTree if err != nil { if err == ErrSkipNode { // skip the remaining entries in this tree - return nil + break } return err From d4ed7c88586cb447e30174b1d127d92924838dd6 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 21 Jan 2024 16:44:00 +0100 Subject: [PATCH 078/158] walker: add tests for leaveDir --- internal/walker/walker_test.go | 53 ++++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/internal/walker/walker_test.go b/internal/walker/walker_test.go index e2d1f866f..0f0009107 100644 --- a/internal/walker/walker_test.go +++ b/internal/walker/walker_test.go @@ -93,12 +93,12 @@ func (t TreeMap) Connections() uint { // checkFunc returns a function suitable for walking the tree to check // something, and a function which will check the final result. -type checkFunc func(t testing.TB) (walker WalkFunc, final func(testing.TB)) +type checkFunc func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) // checkItemOrder ensures that the order of the 'path' arguments is the one passed in as 'want'. func checkItemOrder(want []string) checkFunc { pos := 0 - return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { + return func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) { walker = func(treeID restic.ID, path string, node *restic.Node, err error) error { if err != nil { t.Errorf("error walking %v: %v", path, err) @@ -117,20 +117,24 @@ func checkItemOrder(want []string) checkFunc { return nil } + leaveDir = func(path string) { + _ = walker(restic.ID{}, "leave: "+path, nil, nil) + } + final = func(t testing.TB) { if pos != len(want) { t.Errorf("not enough items returned, want %d, got %d", len(want), pos) } } - return walker, final + return walker, leaveDir, final } } // checkParentTreeOrder ensures that the order of the 'parentID' arguments is the one passed in as 'want'. func checkParentTreeOrder(want []string) checkFunc { pos := 0 - return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { + return func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) { walker = func(treeID restic.ID, path string, node *restic.Node, err error) error { if err != nil { t.Errorf("error walking %v: %v", path, err) @@ -155,7 +159,7 @@ func checkParentTreeOrder(want []string) checkFunc { } } - return walker, final + return walker, nil, final } } @@ -164,7 +168,7 @@ func checkParentTreeOrder(want []string) checkFunc { func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc { var pos int - return func(t testing.TB) (walker WalkFunc, final func(testing.TB)) { + return func(t testing.TB) (walker WalkFunc, leaveDir func(path string), final func(testing.TB)) { walker = func(treeID restic.ID, path string, node *restic.Node, err error) error { if err != nil { t.Errorf("error walking %v: %v", path, err) @@ -188,13 +192,17 @@ func checkSkipFor(skipFor map[string]struct{}, wantPaths []string) checkFunc { return nil } + leaveDir = func(path string) { + _ = walker(restic.ID{}, "leave: "+path, nil, nil) + } + final = func(t testing.TB) { if pos != len(wantPaths) { t.Errorf("wrong number of paths returned, want %d, got %d", len(wantPaths), pos) } } - return walker, final + return walker, leaveDir, final } } @@ -216,6 +224,8 @@ func TestWalker(t *testing.T) { "/foo", "/subdir", "/subdir/subfile", + "leave: /subdir", + "leave: /", }), checkParentTreeOrder([]string{ "a760536a8fd64dd63f8dd95d85d788d71fd1bee6828619350daf6959dcb499a0", // tree / @@ -230,6 +240,7 @@ func TestWalker(t *testing.T) { "/", "/foo", "/subdir", + "leave: /", }, ), checkSkipFor( @@ -260,10 +271,14 @@ func TestWalker(t *testing.T) { "/foo", "/subdir1", "/subdir1/subfile1", + "leave: /subdir1", "/subdir2", "/subdir2/subfile2", "/subdir2/subsubdir2", "/subdir2/subsubdir2/subsubfile3", + "leave: /subdir2/subsubdir2", + "leave: /subdir2", + "leave: /", }), checkParentTreeOrder([]string{ "7a0e59b986cc83167d9fbeeefc54e4629770124c5825d391f7ee0598667fcdf1", // tree / @@ -286,6 +301,9 @@ func TestWalker(t *testing.T) { "/subdir2/subfile2", "/subdir2/subsubdir2", "/subdir2/subsubdir2/subsubfile3", + "leave: /subdir2/subsubdir2", + "leave: /subdir2", + "leave: /", }, ), checkSkipFor( @@ -299,6 +317,8 @@ func TestWalker(t *testing.T) { "/subdir2", "/subdir2/subfile2", "/subdir2/subsubdir2", + "leave: /subdir2", + "leave: /", }, ), checkSkipFor( @@ -307,6 +327,7 @@ func TestWalker(t *testing.T) { }, []string{ "/", "/foo", + "leave: /", }, ), }, @@ -339,15 +360,19 @@ func TestWalker(t *testing.T) { "/subdir1/subfile1", "/subdir1/subfile2", "/subdir1/subfile3", + "leave: /subdir1", "/subdir2", "/subdir2/subfile1", "/subdir2/subfile2", "/subdir2/subfile3", + "leave: /subdir2", "/subdir3", "/subdir3/subfile1", "/subdir3/subfile2", "/subdir3/subfile3", + "leave: /subdir3", "/zzz other", + "leave: /", }), checkParentTreeOrder([]string{ "c2efeff7f217a4dfa12a16e8bb3cefedd37c00873605c29e5271c6061030672f", // tree / @@ -385,13 +410,20 @@ func TestWalker(t *testing.T) { checkItemOrder([]string{ "/", "/subdir1", + "leave: /subdir1", "/subdir2", + "leave: /subdir2", "/subdir3", "/subdir3/file", + "leave: /subdir3", "/subdir4", "/subdir4/file", + "leave: /subdir4", "/subdir5", + "leave: /subdir5", "/subdir6", + "leave: /subdir6", + "leave: /", }), }, }, @@ -405,8 +437,11 @@ func TestWalker(t *testing.T) { ctx, cancel := context.WithCancel(context.TODO()) defer cancel() - fn, last := check(t) - err := Walk(ctx, repo, root, WalkVisitor{ProcessNode: fn}) + fn, leaveDir, last := check(t) + err := Walk(ctx, repo, root, WalkVisitor{ + ProcessNode: fn, + LeaveDir: leaveDir, + }) if err != nil { t.Error(err) } From 10e71af759679b2c51dd1df2f58181925977e31c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 21 Jan 2024 17:09:54 +0100 Subject: [PATCH 079/158] describe ls command in docs --- doc/045_working_with_repos.rst | 71 ++++++++++++++++++++++++++++++++++ doc/075_scripting.rst | 2 + 2 files changed, 73 insertions(+) diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index d74c9c240..48e5985dc 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -82,6 +82,76 @@ Furthermore you can group the output by the same filters (host, paths, tags): 1 snapshots +Listing files in a snapshot +=========================== + +To get a list of the files in a specific snapshot you can use the ``ls`` command: + +.. code-block:: console + + $ restic ls 073a90db + + snapshot 073a90db of [/home/user/work.txt] filtered by [] at 2024-01-21 16:51:18.474558607 +0100 CET): + /home + /home/user + /home/user/work.txt + +The special snapshot ID ``latest`` can be used to list files and directories of the latest snapshot in the repository. +The ``--host`` flag can be used in conjunction to select the latest snapshot originating from a certain host only. + +.. code-block:: console + + $ restic ls --host kasimir latest + + snapshot 073a90db of [/home/user/work.txt] filtered by [] at 2024-01-21 16:51:18.474558607 +0100 CET): + /home + /home/user + /home/user/work.txt + +By default, ``ls`` prints all files in a snapshot. + +File listings can optionally be filtered by directories. Any positional arguments after the snapshot ID are interpreted +as absolute directory paths, and only files inside those directories will be listed. Files in subdirectories are not +listed when filtering by directories. If the ``--recursive`` flag is used, then subdirectories are also included. +Any directory paths specified must be absolute (starting with a path separator); paths use the forward slash '/' +as separator. + +.. code-block:: console + + $ restic ls latest /home + + snapshot 073a90db of [/home/user/work.txt] filtered by [/home] at 2024-01-21 16:51:18.474558607 +0100 CET): + /home + /home/user + +.. code-block:: console + + $ restic ls --recursive latest /home + + snapshot 073a90db of [/home/user/work.txt] filtered by [/home] at 2024-01-21 16:51:18.474558607 +0100 CET): + /home + /home/user + /home/user/work.txt + +To show more details about the files in a snapshot, you can use the ``--long`` option. The colums include +file permissions, UID, GID, file size, modification time and file path. For scripting usage, the +``ls`` command supports the ``--json`` flag; the JSON output format is described at :ref:`ls json`. + +.. code-block:: console + + $ restic ls --long latest + + snapshot 073a90db of [/home/user/work.txt] filtered by [] at 2024-01-21 16:51:18.474558607 +0100 CET): + drwxr-xr-x 0 0 0 2024-01-21 16:50:52 /home + drwxr-xr-x 0 0 0 2024-01-21 16:51:03 /home/user + -rw-r--r-- 0 0 18 2024-01-21 16:51:03 /home/user/work.txt + +NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories. The ``ls`` command supports +outputting information about a snapshot in the NCDU format using the ``--ncdu`` option. + +You can use it as follows: ``restic ls latest --ncdu | ncdu -f -`` + + Copying snapshots between repositories ====================================== @@ -242,6 +312,7 @@ Currently, rewriting the hostname and the time of the backup is supported. This is possible using the ``rewrite`` command with the option ``--new-host`` followed by the desired new hostname or the option ``--new-time`` followed by the desired new timestamp. .. code-block:: console + $ restic rewrite --new-host newhost --new-time "1999-01-01 11:11:11" repository b7dbade3 opened (version 2, compression level auto) diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index f46572209..7279ee614 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -409,6 +409,8 @@ The ``key list`` command returns an array of objects with the following structur +--------------+------------------------------------+ +.. _ls json: + ls -- From a09d51d96cc1aa2f86400325621d29f5ae1479e2 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 27 Jan 2024 13:42:29 +0100 Subject: [PATCH 080/158] backup: clarify that --ignore-inode also ignores ctime --- cmd/restic/cmd_backup.go | 2 +- doc/manual_rest.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index 2ea2a4ec5..8101aa40b 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -126,7 +126,7 @@ func init() { f.StringArrayVar(&backupOptions.FilesFromRaw, "files-from-raw", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)") f.StringVar(&backupOptions.TimeStamp, "time", "", "`time` of the backup (ex. '2012-11-01 22:08:41') (default: now)") f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories") - f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files") + f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number and ctime changes when checking for modified files") f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files") f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done") f.BoolVar(&backupOptions.NoScan, "no-scan", false, "do not run scanner to estimate size of backup") diff --git a/doc/manual_rest.rst b/doc/manual_rest.rst index 1ccd9268d..bf9554e04 100644 --- a/doc/manual_rest.rst +++ b/doc/manual_rest.rst @@ -112,7 +112,7 @@ command: --iexclude pattern same as --exclude pattern but ignores the casing of filenames --iexclude-file file same as --exclude-file but ignores casing of filenames in patterns --ignore-ctime ignore ctime changes when checking for modified files - --ignore-inode ignore inode number changes when checking for modified files + --ignore-inode ignore inode number and ctime changes when checking for modified files --no-scan do not run scanner to estimate size of backup -x, --one-file-system exclude other file systems, don't cross filesystem boundaries and subvolumes --parent snapshot use this parent snapshot (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time) From 765729d009d624c0dd0ea4a8dd88fde6010bccd1 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 27 Jan 2024 15:33:49 +0100 Subject: [PATCH 081/158] key: return an error if subcommand is unknown --- cmd/restic/cmd_key.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index 43aa07276..a4de4f682 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -250,9 +250,9 @@ func runKey(ctx context.Context, gopts GlobalOptions, opts KeyOptions, args []st } return changePassword(ctx, repo, gopts, opts.NewPasswordFile) + default: + return errors.Fatal("invalid operation") } - - return nil } func loadPasswordFromFile(pwdFile string) (string, error) { From 5dca8a70d56034c3c4c4cdb6d42c04f95e83a13d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 27 Jan 2024 15:48:24 +0100 Subject: [PATCH 082/158] ls: include standard `message_type` field in output --- changelog/unreleased/pull-4664 | 8 ++ cmd/restic/cmd_ls.go | 20 ++-- cmd/restic/cmd_ls_test.go | 10 +- cmd/restic/cmd_mount_integration_test.go | 6 -- doc/075_scripting.rst | 111 ++++++++++++----------- 5 files changed, 81 insertions(+), 74 deletions(-) create mode 100644 changelog/unreleased/pull-4664 diff --git a/changelog/unreleased/pull-4664 b/changelog/unreleased/pull-4664 new file mode 100644 index 000000000..74196cd9b --- /dev/null +++ b/changelog/unreleased/pull-4664 @@ -0,0 +1,8 @@ +Enhancement: `ls` uses `message_type` field to distinguish JSON messages + +The `ls` command was the only command that used the `struct_type` field to determine +the message type in the JSON output format. Now, the JSON output of the +`ls` command also includes the `message_type`. The `struct_type` field is +still included, but it deprecated. + +https://github.com/restic/restic/pull/4664 diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index f412546ae..b0246625e 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -83,16 +83,18 @@ type jsonLsPrinter struct { func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) { type lsSnapshot struct { *restic.Snapshot - ID *restic.ID `json:"id"` - ShortID string `json:"short_id"` - StructType string `json:"struct_type"` // "snapshot" + ID *restic.ID `json:"id"` + ShortID string `json:"short_id"` + MessageType string `json:"message_type"` // "snapshot" + StructType string `json:"struct_type"` // "snapshot", deprecated } err := p.enc.Encode(lsSnapshot{ - Snapshot: sn, - ID: sn.ID(), - ShortID: sn.ID().Str(), - StructType: "snapshot", + Snapshot: sn, + ID: sn.ID(), + ShortID: sn.ID().Str(), + MessageType: "snapshot", + StructType: "snapshot", }) if err != nil { Warnf("JSON encode failed: %v\n", err) @@ -121,7 +123,8 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { AccessTime time.Time `json:"atime,omitempty"` ChangeTime time.Time `json:"ctime,omitempty"` Inode uint64 `json:"inode,omitempty"` - StructType string `json:"struct_type"` // "node" + MessageType string `json:"message_type"` // "node" + StructType string `json:"struct_type"` // "node", deprecated size uint64 // Target for Size pointer. }{ @@ -137,6 +140,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error { AccessTime: node.AccessTime, ChangeTime: node.ChangeTime, Inode: node.Inode, + MessageType: "node", StructType: "node", } // Always print size for regular files, even when empty, diff --git a/cmd/restic/cmd_ls_test.go b/cmd/restic/cmd_ls_test.go index 41c235eab..828b2920e 100644 --- a/cmd/restic/cmd_ls_test.go +++ b/cmd/restic/cmd_ls_test.go @@ -87,11 +87,11 @@ var lsTestNodes = []lsTestNode{ func TestLsNodeJSON(t *testing.T) { for i, expect := range []string{ - `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, - `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, - `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, - `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`, - `{"name":"sticky","type":"dir","path":"/some/sticky","uid":0,"gid":0,"mode":2161115629,"permissions":"dugtrwxr-xr-x","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`, + `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","message_type":"node","struct_type":"node"}`, + `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","message_type":"node","struct_type":"node"}`, + `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","message_type":"node","struct_type":"node"}`, + `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","message_type":"node","struct_type":"node"}`, + `{"name":"sticky","type":"dir","path":"/some/sticky","uid":0,"gid":0,"mode":2161115629,"permissions":"dugtrwxr-xr-x","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","message_type":"node","struct_type":"node"}`, } { c := lsTestNodes[i] buf := new(bytes.Buffer) diff --git a/cmd/restic/cmd_mount_integration_test.go b/cmd/restic/cmd_mount_integration_test.go index 1b069d582..d2025a395 100644 --- a/cmd/restic/cmd_mount_integration_test.go +++ b/cmd/restic/cmd_mount_integration_test.go @@ -12,7 +12,6 @@ import ( "testing" "time" - "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/repository" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" @@ -160,11 +159,6 @@ func TestMount(t *testing.T) { t.Skip("Skipping fuse tests") } - debugEnabled := debug.TestLogToStderr(t) - if debugEnabled { - defer debug.TestDisableLog(t) - } - env, cleanup := withTestEnvironment(t) // must list snapshots more than once env.gopts.backendTestHook = nil diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index 7279ee614..fda4b2d53 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -75,9 +75,6 @@ Several commands, in particular long running ones or those that generate a large use a format also known as JSON lines. It consists of a stream of new-line separated JSON messages. You can determine the nature of the message using the ``message_type`` field. -As an exception, the ``ls`` command uses the field ``struct_type`` instead. - - backup ------ @@ -420,63 +417,67 @@ As an exception, the ``struct_type`` field is used to determine the message type snapshot ^^^^^^^^ -+----------------+--------------------------------------------------+ -| ``struct_type``| Always "snapshot" | -+----------------+--------------------------------------------------+ -| ``time`` | Timestamp of when the backup was started | -+----------------+--------------------------------------------------+ -| ``parent`` | ID of the parent snapshot | -+----------------+--------------------------------------------------+ -| ``tree`` | ID of the root tree blob | -+----------------+--------------------------------------------------+ -| ``paths`` | List of paths included in the backup | -+----------------+--------------------------------------------------+ -| ``hostname`` | Hostname of the backed up machine | -+----------------+--------------------------------------------------+ -| ``username`` | Username the backup command was run as | -+----------------+--------------------------------------------------+ -| ``uid`` | ID of owner | -+----------------+--------------------------------------------------+ -| ``gid`` | ID of group | -+----------------+--------------------------------------------------+ -| ``excludes`` | List of paths and globs excluded from the backup | -+----------------+--------------------------------------------------+ -| ``tags`` | List of tags for the snapshot in question | -+----------------+--------------------------------------------------+ -| ``id`` | Snapshot ID | -+----------------+--------------------------------------------------+ -| ``short_id`` | Snapshot ID, short form | -+----------------+--------------------------------------------------+ ++------------------+--------------------------------------------------+ +| ``message_type`` | Always "snapshot" | ++------------------+--------------------------------------------------+ +| ``struct_type`` | Always "snapshot" (deprecated) | ++------------------+--------------------------------------------------+ +| ``time`` | Timestamp of when the backup was started | ++------------------+--------------------------------------------------+ +| ``parent`` | ID of the parent snapshot | ++------------------+--------------------------------------------------+ +| ``tree`` | ID of the root tree blob | ++------------------+--------------------------------------------------+ +| ``paths`` | List of paths included in the backup | ++------------------+--------------------------------------------------+ +| ``hostname`` | Hostname of the backed up machine | ++------------------+--------------------------------------------------+ +| ``username`` | Username the backup command was run as | ++------------------+--------------------------------------------------+ +| ``uid`` | ID of owner | ++------------------+--------------------------------------------------+ +| ``gid`` | ID of group | ++------------------+--------------------------------------------------+ +| ``excludes`` | List of paths and globs excluded from the backup | ++------------------+--------------------------------------------------+ +| ``tags`` | List of tags for the snapshot in question | ++------------------+--------------------------------------------------+ +| ``id`` | Snapshot ID | ++------------------+--------------------------------------------------+ +| ``short_id`` | Snapshot ID, short form | ++------------------+--------------------------------------------------+ node ^^^^ -+-----------------+--------------------------+ -| ``struct_type`` | Always "node" | -+-----------------+--------------------------+ -| ``name`` | Node name | -+-----------------+--------------------------+ -| ``type`` | Node type | -+-----------------+--------------------------+ -| ``path`` | Node path | -+-----------------+--------------------------+ -| ``uid`` | UID of node | -+-----------------+--------------------------+ -| ``gid`` | GID of node | -+-----------------+--------------------------+ -| ``size`` | Size in bytes | -+-----------------+--------------------------+ -| ``mode`` | Node mode | -+-----------------+--------------------------+ -| ``atime`` | Node access time | -+-----------------+--------------------------+ -| ``mtime`` | Node modification time | -+-----------------+--------------------------+ -| ``ctime`` | Node creation time | -+-----------------+--------------------------+ -| ``inode`` | Inode number of node | -+-----------------+--------------------------+ ++------------------+----------------------------+ +| ``message_type`` | Always "node" | ++------------------+----------------------------+ +| ``struct_type`` | Always "node" (deprecated) | ++------------------+----------------------------+ +| ``name`` | Node name | ++------------------+----------------------------+ +| ``type`` | Node type | ++------------------+----------------------------+ +| ``path`` | Node path | ++------------------+----------------------------+ +| ``uid`` | UID of node | ++------------------+----------------------------+ +| ``gid`` | GID of node | ++------------------+----------------------------+ +| ``size`` | Size in bytes | ++------------------+----------------------------+ +| ``mode`` | Node mode | ++------------------+----------------------------+ +| ``atime`` | Node access time | ++------------------+----------------------------+ +| ``mtime`` | Node modification time | ++------------------+----------------------------+ +| ``ctime`` | Node creation time | ++------------------+----------------------------+ +| ``inode`` | Inode number of node | ++------------------+----------------------------+ restore From 8fbe32837148d18294d092d85e6dfe62e76b3214 Mon Sep 17 00:00:00 2001 From: Lionel Sausin Date: Thu, 25 Jan 2024 10:12:11 +0100 Subject: [PATCH 083/158] Properly report the ID of newly added keys Other commands like key list and key remove show the key's ID. Showing the ID here lets users easily reuse the ID as a key hint for subsequent commands. In particular, a key hint is needed when the repository has many keys - otherwise opening the repository may fail with "Fatal: maximum number of keys reached" even when a proper password is provided. Fixes #4656 --- changelog/unreleased/issue-4656 | 7 +++++++ cmd/restic/cmd_key.go | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/issue-4656 diff --git a/changelog/unreleased/issue-4656 b/changelog/unreleased/issue-4656 new file mode 100644 index 000000000..8d16f0b48 --- /dev/null +++ b/changelog/unreleased/issue-4656 @@ -0,0 +1,7 @@ +Bugfix: Properly report the ID of newly added keys + +`restic key add` now reports the ID of a newly added key. This simplifies +selecting a specific key using the `--key-hint key` option. + +https://github.com/restic/restic/issues/4656 +https://github.com/restic/restic/pull/4657 diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index 8309f8382..acd4a029e 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -142,7 +142,7 @@ func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOption return err } - Verbosef("saved new key as %s\n", id) + Verbosef("saved new key with ID %s\n", id.ID()) return nil } From ba136ff60c4d4d23abde2340b5aa4616820c80d8 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Wed, 31 Jan 2024 21:48:37 +0100 Subject: [PATCH 084/158] rewrite: fix typo in dry-run output --- cmd/restic/cmd_rewrite.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index d55e6137b..8dad492af 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -209,7 +209,7 @@ func filterAndReplaceSnapshot(ctx context.Context, repo restic.Repository, sn *r } if newMetadata != nil && newMetadata.Hostname != "" { - Verbosef("would set time to %s\n", newMetadata.Hostname) + Verbosef("would set hostname to %s\n", newMetadata.Hostname) } return true, nil From b07afa9b020438a6a547fcb9bafc99af02eb569f Mon Sep 17 00:00:00 2001 From: Joram Berger Date: Wed, 31 Jan 2024 23:40:21 +0100 Subject: [PATCH 085/158] Link to Go Match syntax directly in 040_backup.rst The docs linked to filepath.Glob (which is used) but the syntax is described in the documentation of filepath.Match. So it makes sense to link that directly. --- doc/040_backup.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/040_backup.rst b/doc/040_backup.rst index ae834491e..550957eeb 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -381,8 +381,8 @@ contains one *pattern* per line. The file must be encoded as UTF-8, or UTF-16 with a byte-order mark. Leading and trailing whitespace is removed from the patterns. Empty lines and lines starting with a ``#`` are ignored and each pattern is expanded when read, such that special characters in it are expanded -using the Go function `filepath.Glob `__ -- please see its documentation for the syntax you can use in the patterns. +according to the syntax described in the documentation of the Go function +`filepath.Match `__. The argument passed to ``--files-from-verbatim`` must be the name of a text file that contains one *path* per line, e.g. as generated by GNU ``find`` with the From 7f5ea511bca114fc68362ade547ceb3712676164 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 01:51:32 +0000 Subject: [PATCH 086/158] build(deps): bump github.com/Azure/azure-sdk-for-go/sdk/storage/azblob Bumps [github.com/Azure/azure-sdk-for-go/sdk/storage/azblob](https://github.com/Azure/azure-sdk-for-go) from 1.2.0 to 1.2.1. - [Release notes](https://github.com/Azure/azure-sdk-for-go/releases) - [Changelog](https://github.com/Azure/azure-sdk-for-go/blob/main/documentation/release.md) - [Commits](https://github.com/Azure/azure-sdk-for-go/compare/v1.2...sdk/azidentity/v1.2.1) --- updated-dependencies: - dependency-name: github.com/Azure/azure-sdk-for-go/sdk/storage/azblob dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index f07fc10ee..078dcb3f1 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ require ( cloud.google.com/go/storage v1.34.0 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 github.com/Backblaze/blazer v0.6.1 github.com/anacrolix/fuse v0.2.0 github.com/cenkalti/backoff/v4 v4.2.1 diff --git a/go.sum b/go.sum index 01e58cccb..73562d934 100644 --- a/go.sum +++ b/go.sum @@ -15,9 +15,9 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZM github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 h1:gggzg0SUMs6SQbEw+3LoSsYf9YMjkupeAnHMX8O9mmY= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0/go.mod h1:+6KLcKIVgxoBDMqMO/Nvy7bZ9a0nbU3I1DtFQK3YvB4= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.5.0 h1:AifHbc4mg0x9zW52WOpKbsHaDKuRhlI7TVl47thgQ70= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 h1:AMf7YbZOZIW5b66cXNHMWWT/zkjhz5+a+k/3x40EO7E= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1/go.mod h1:uwfk06ZBcvL/g4VHNjurPfVln9NMbsk2XIZxJ+hu81k= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 h1:hVeq+yCyUi+MsoO/CU95yqCIcdzra5ovzk8Q2BBpV2M= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/Backblaze/blazer v0.6.1 h1:xC9HyC7OcxRzzmtfRiikIEvq4HZYWjU6caFwX2EXw1s= From 552f01662b8a5267e1890015d8f713998fb58e75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 01:53:12 +0000 Subject: [PATCH 087/158] build(deps): bump golang.org/x/oauth2 from 0.15.0 to 0.16.0 Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from 0.15.0 to 0.16.0. - [Commits](https://github.com/golang/oauth2/compare/v0.15.0...v0.16.0) --- updated-dependencies: - dependency-name: golang.org/x/oauth2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index f07fc10ee..80e14ad54 100644 --- a/go.mod +++ b/go.mod @@ -25,12 +25,12 @@ require ( github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 go.uber.org/automaxprocs v1.5.3 - golang.org/x/crypto v0.17.0 - golang.org/x/net v0.19.0 - golang.org/x/oauth2 v0.15.0 + golang.org/x/crypto v0.18.0 + golang.org/x/net v0.20.0 + golang.org/x/oauth2 v0.16.0 golang.org/x/sync v0.5.0 - golang.org/x/sys v0.15.0 - golang.org/x/term v0.15.0 + golang.org/x/sys v0.16.0 + golang.org/x/term v0.16.0 golang.org/x/text v0.14.0 golang.org/x/time v0.5.0 google.golang.org/api v0.149.0 diff --git a/go.sum b/go.sum index 01e58cccb..65e27b32b 100644 --- a/go.sum +++ b/go.sum @@ -183,8 +183,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -202,11 +202,11 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= -golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -229,13 +229,13 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= From 207a4a5e8e68218a0a3d964f97930d98564371c2 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 2 Feb 2024 20:10:29 +0100 Subject: [PATCH 088/158] Downgrade klauspost/compress to fix data corruption at max. compression --- go.mod | 2 ++ go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index f07fc10ee..970770a90 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,8 @@ require ( google.golang.org/api v0.149.0 ) +replace github.com/klauspost/compress => github.com/klauspost/compress v1.17.2 + require ( cloud.google.com/go v0.110.9 // indirect cloud.google.com/go/compute v1.23.1 // indirect diff --git a/go.sum b/go.sum index 01e58cccb..129eafed2 100644 --- a/go.sum +++ b/go.sum @@ -108,8 +108,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= From cf7cad11defcdde7c6e84f094c388c344670268b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 2 Feb 2024 20:40:57 +0000 Subject: [PATCH 089/158] build(deps): bump cloud.google.com/go/storage from 1.34.0 to 1.37.0 Bumps [cloud.google.com/go/storage](https://github.com/googleapis/google-cloud-go) from 1.34.0 to 1.37.0. - [Release notes](https://github.com/googleapis/google-cloud-go/releases) - [Changelog](https://github.com/googleapis/google-cloud-go/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-cloud-go/compare/pubsub/v1.34.0...spanner/v1.37.0) --- updated-dependencies: - dependency-name: cloud.google.com/go/storage dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 31 +++++++++++++++++----------- go.sum | 65 +++++++++++++++++++++++++++++++++++++--------------------- 2 files changed, 61 insertions(+), 35 deletions(-) diff --git a/go.mod b/go.mod index 57c5ca470..1a4a3445e 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/restic/restic require ( - cloud.google.com/go/storage v1.34.0 + cloud.google.com/go/storage v1.37.0 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 @@ -28,24 +28,27 @@ require ( golang.org/x/crypto v0.18.0 golang.org/x/net v0.20.0 golang.org/x/oauth2 v0.16.0 - golang.org/x/sync v0.5.0 + golang.org/x/sync v0.6.0 golang.org/x/sys v0.16.0 golang.org/x/term v0.16.0 golang.org/x/text v0.14.0 golang.org/x/time v0.5.0 - google.golang.org/api v0.149.0 + google.golang.org/api v0.157.0 ) require ( - cloud.google.com/go v0.110.9 // indirect - cloud.google.com/go/compute v1.23.1 // indirect + cloud.google.com/go v0.112.0 // indirect + cloud.google.com/go/compute v1.23.3 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.3 // indirect + cloud.google.com/go/iam v1.1.5 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/felixge/fgprof v0.9.3 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -68,13 +71,17 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect + go.opentelemetry.io/otel v1.21.0 // indirect + go.opentelemetry.io/otel/metric v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.21.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect - google.golang.org/grpc v1.59.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240122161410-6c6643bf1457 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac // indirect + google.golang.org/grpc v1.60.1 // indirect + google.golang.org/protobuf v1.32.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index a4e313a53..291b570f8 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,14 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.110.9 h1:e7ITSqGFFk4rbz/JFIqZh3G4VEHguhAL4BQcFlWtU68= -cloud.google.com/go v0.110.9/go.mod h1:rpxevX/0Lqvlbc88b7Sc1SPNdyK1riNBTUU6JXhYNpM= -cloud.google.com/go/compute v1.23.1 h1:V97tBoDaZHb6leicZ1G6DLK2BAaZLJ/7+9BB/En3hR0= -cloud.google.com/go/compute v1.23.1/go.mod h1:CqB3xpmPKKt3OJpW2ndFIXnA9A4xAy/F3Xp1ixncW78= +cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= +cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= +cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/iam v1.1.3 h1:18tKG7DzydKWUnLjonWcJO6wjSCAtzh4GcRKlH/Hrzc= -cloud.google.com/go/iam v1.1.3/go.mod h1:3khUlaBXfPKKe7huYgEpDn6FtgRyMEqbkvBxrQyY5SE= -cloud.google.com/go/storage v1.34.0 h1:9KHBBTbaHPsNxO043SFmH3pMojjZiW+BFl9H41L7xjk= -cloud.google.com/go/storage v1.34.0/go.mod h1:Eji+S0CCQebjsiXxyIvPItC3BN3zWsdJjWfHfoLblgY= +cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= +cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= +cloud.google.com/go/storage v1.37.0 h1:WI8CsaFO8Q9KjPVtsZ5Cmi0dXV25zMoX0FklT7c3Jm4= +cloud.google.com/go/storage v1.37.0/go.mod h1:i34TiT2IhiNDmcj65PqwCjcoUX7Z5pLzS8DEmoiFq1k= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= @@ -36,6 +36,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= @@ -54,8 +55,16 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g= github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= @@ -176,6 +185,17 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -212,8 +232,8 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -257,9 +277,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -google.golang.org/api v0.149.0 h1:b2CqT6kG+zqJIVKRQ3ELJVLN1PwHZ6DJ3dW8yl82rgY= -google.golang.org/api v0.149.0/go.mod h1:Mwn1B7JTXrzXtnvmzQE2BD6bYZQ8DShKZDZbeN9I7qI= +google.golang.org/api v0.157.0 h1:ORAeqmbrrozeyw5NjnMxh7peHO0UzV4wWYSwZeCUb20= +google.golang.org/api v0.157.0/go.mod h1:+z4v4ufbZ1WEpld6yMGHyggs+PmAHiaLNj5ytP3N01g= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= @@ -267,19 +286,19 @@ google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJ google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA= -google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI= -google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k= -google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= +google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac h1:ZL/Teoy/ZGnzyrqK/Optxxp2pmVh+fmJ97slxSRyzUg= +google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:+Rvu7ElI+aLzyDQhpHMFMMltsD6m7nqpuWDd2CwJw3k= +google.golang.org/genproto/googleapis/api v0.0.0-20240122161410-6c6643bf1457 h1:KHBtwE+eQc3+NxpjmRFlQ3pJQ2FNnhhgB9xOV8kyBuU= +google.golang.org/genproto/googleapis/api v0.0.0-20240122161410-6c6643bf1457/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac h1:nUQEQmH/csSvFECKYRv6HWEyypysidKl2I6Qpsglq/0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -291,8 +310,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= From bb92b487f72171c28f4a9fbd8662e19895a78f12 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 17:35:46 +0100 Subject: [PATCH 090/158] repository: fix repack test --- internal/repository/repair_pack_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/repository/repair_pack_test.go b/internal/repository/repair_pack_test.go index 6b20dbffb..3f9477945 100644 --- a/internal/repository/repair_pack_test.go +++ b/internal/repository/repair_pack_test.go @@ -38,17 +38,17 @@ func TestRepairBrokenPack(t *testing.T) { func testRepairBrokenPack(t *testing.T, version uint) { tests := []struct { name string - damage func(repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) + damage func(t *testing.T, repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) }{ { "valid pack", - func(repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { + func(t *testing.T, repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { return packsBefore, restic.NewBlobSet() }, }, { "broken pack", - func(repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { + func(t *testing.T, repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { wrongBlob := createRandomWrongBlob(t, repo) damagedPacks := findPacksForBlobs(t, repo, restic.NewBlobSet(wrongBlob)) return damagedPacks, restic.NewBlobSet(wrongBlob) @@ -56,7 +56,7 @@ func testRepairBrokenPack(t *testing.T, version uint) { }, { "partially broken pack", - func(repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { + func(t *testing.T, repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { // damage one of the pack files damagedID := packsBefore.List()[0] replaceFile(t, repo, backend.Handle{Type: backend.PackFile, Name: damagedID.String()}, @@ -79,7 +79,7 @@ func testRepairBrokenPack(t *testing.T, version uint) { }, }, { "truncated pack", - func(repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { + func(t *testing.T, repo restic.Repository, packsBefore restic.IDSet) (restic.IDSet, restic.BlobSet) { // damage one of the pack files damagedID := packsBefore.List()[0] replaceFile(t, repo, backend.Handle{Type: backend.PackFile, Name: damagedID.String()}, @@ -112,7 +112,7 @@ func testRepairBrokenPack(t *testing.T, version uint) { packsBefore := listPacks(t, repo) blobsBefore := listBlobs(repo) - toRepair, damagedBlobs := test.damage(repo, packsBefore) + toRepair, damagedBlobs := test.damage(t, repo, packsBefore) rtest.OK(t, repository.RepairPacks(context.TODO(), repo, toRepair, &progress.NoopPrinter{})) // reload index From 16e3f79e8b3cf0d9672e6cca2f0760e45d3d0624 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 17:47:36 +0100 Subject: [PATCH 091/158] repository: make repo.Options configurable for test repos --- internal/archiver/archiver_test.go | 2 +- internal/migrations/upgrade_repo_v2_test.go | 2 +- internal/repository/fuzz_test.go | 3 +-- internal/repository/testing.go | 9 +++++---- internal/restic/lock_test.go | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index c6daed5bb..46ef44251 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -1880,7 +1880,7 @@ func TestArchiverContextCanceled(t *testing.T) { }) // Ensure that the archiver itself reports the canceled context and not just the backend - repo := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0) + repo := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0, repository.Options{}) back := restictest.Chdir(t, tempdir) defer back() diff --git a/internal/migrations/upgrade_repo_v2_test.go b/internal/migrations/upgrade_repo_v2_test.go index 40153d3ca..845d20e92 100644 --- a/internal/migrations/upgrade_repo_v2_test.go +++ b/internal/migrations/upgrade_repo_v2_test.go @@ -69,7 +69,7 @@ func TestUpgradeRepoV2Failure(t *testing.T) { Backend: be, } - repo := repository.TestRepositoryWithBackend(t, be, 1) + repo := repository.TestRepositoryWithBackend(t, be, 1, repository.Options{}) if repo.Config().Version != 1 { t.Fatal("test repo has wrong version") } diff --git a/internal/repository/fuzz_test.go b/internal/repository/fuzz_test.go index b4036288c..80372f8e0 100644 --- a/internal/repository/fuzz_test.go +++ b/internal/repository/fuzz_test.go @@ -4,7 +4,6 @@ import ( "context" "testing" - "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/restic" "golang.org/x/sync/errgroup" ) @@ -19,7 +18,7 @@ func FuzzSaveLoadBlob(f *testing.F) { } id := restic.Hash(blob) - repo := TestRepositoryWithBackend(t, mem.New(), 2) + repo := TestRepositoryWithVersion(t, 2) var wg errgroup.Group repo.StartPackUploader(context.TODO(), &wg) diff --git a/internal/repository/testing.go b/internal/repository/testing.go index d79137425..dbbdbeb07 100644 --- a/internal/repository/testing.go +++ b/internal/repository/testing.go @@ -44,7 +44,7 @@ const TestChunkerPol = chunker.Pol(0x3DA3358B4DC173) // TestRepositoryWithBackend returns a repository initialized with a test // password. If be is nil, an in-memory backend is used. A constant polynomial // is used for the chunker and low-security test parameters. -func TestRepositoryWithBackend(t testing.TB, be backend.Backend, version uint) restic.Repository { +func TestRepositoryWithBackend(t testing.TB, be backend.Backend, version uint, opts Options) restic.Repository { t.Helper() TestUseLowSecurityKDFParameters(t) restic.TestDisableCheckPolynomial(t) @@ -53,7 +53,7 @@ func TestRepositoryWithBackend(t testing.TB, be backend.Backend, version uint) r be = TestBackend(t) } - repo, err := New(be, Options{}) + repo, err := New(be, opts) if err != nil { t.Fatalf("TestRepository(): new repo failed: %v", err) } @@ -79,6 +79,7 @@ func TestRepository(t testing.TB) restic.Repository { func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository { t.Helper() dir := os.Getenv("RESTIC_TEST_REPO") + opts := Options{} if dir != "" { _, err := os.Stat(dir) if err != nil { @@ -86,7 +87,7 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository { if err != nil { t.Fatalf("error creating local backend at %v: %v", dir, err) } - return TestRepositoryWithBackend(t, be, version) + return TestRepositoryWithBackend(t, be, version, opts) } if err == nil { @@ -94,7 +95,7 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository { } } - return TestRepositoryWithBackend(t, nil, version) + return TestRepositoryWithBackend(t, nil, version, opts) } // TestOpenLocal opens a local repository. diff --git a/internal/restic/lock_test.go b/internal/restic/lock_test.go index 13b66a432..0d282aaf7 100644 --- a/internal/restic/lock_test.go +++ b/internal/restic/lock_test.go @@ -66,7 +66,7 @@ func (be *failLockLoadingBackend) Load(ctx context.Context, h backend.Handle, le func TestMultipleLockFailure(t *testing.T) { be := &failLockLoadingBackend{Backend: mem.New()} - repo := repository.TestRepositoryWithBackend(t, be, 0) + repo := repository.TestRepositoryWithBackend(t, be, 0, repository.Options{}) restic.TestSetLockTimeout(t, 5*time.Millisecond) lock1, err := restic.NewLock(context.TODO(), repo) From c01a0c6da71383a4aae74174e82c33a0e45d8ee6 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 2 Feb 2024 21:15:39 +0100 Subject: [PATCH 092/158] backup: verify blobs before upload This only covers the blobs themselves, the pack header is not verified so far. Unpacked files are also not covered by the integrity check. --- internal/repository/repository.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 9db83a4df..fadb120ce 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -423,6 +423,11 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data // encrypt blob ciphertext = r.key.Seal(ciphertext, nonce, data, nil) + if err := r.verifyCiphertext(ciphertext, uncompressedLength, id); err != nil { + // FIXME call to action + return 0, fmt.Errorf("detected data corruption while saving blob %v: %w", id, err) + } + // find suitable packer and add blob var pm *packerManager @@ -438,6 +443,27 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data return pm.SaveBlob(ctx, t, id, ciphertext, uncompressedLength) } +func (r *Repository) verifyCiphertext(buf []byte, uncompressedLength int, id restic.ID) error { + nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] + plaintext, err := r.key.Open(nil, nonce, ciphertext, nil) + if err != nil { + return fmt.Errorf("decryption failed: %w", err) + } + if uncompressedLength != 0 { + // DecodeAll will allocate a slice if it is not large enough since it + // knows the decompressed size (because we're using EncodeAll) + plaintext, err = r.getZstdDecoder().DecodeAll(plaintext, nil) + if err != nil { + return fmt.Errorf("decompression failed: %w", err) + } + } + if !restic.Hash(plaintext).Equal(id) { + return errors.New("hash mismatch") + } + + return nil +} + func (r *Repository) compressUnpacked(p []byte) ([]byte, error) { // compression is only available starting from version 2 if r.cfg.Version < 2 { From 30a84e90035c53079d422ecf4ee0346e6a6c2886 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 17:30:58 +0100 Subject: [PATCH 093/158] backup: verify unpacked files before upload --- internal/repository/repository.go | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index fadb120ce..2584f42c7 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -500,7 +500,8 @@ func (r *Repository) decompressUnpacked(p []byte) ([]byte, error) { // SaveUnpacked encrypts data and stores it in the backend. Returned is the // storage hash. -func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []byte) (id restic.ID, err error) { +func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf []byte) (id restic.ID, err error) { + p := buf if t != restic.ConfigFile { p, err = r.compressUnpacked(p) if err != nil { @@ -515,6 +516,11 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by ciphertext = r.key.Seal(ciphertext, nonce, p, nil) + if err := r.verifyUnpacked(ciphertext, t, buf); err != nil { + // FIXME call to action + return restic.ID{}, fmt.Errorf("detected data corruption while saving file of type %v: %w", t, err) + } + if t == restic.ConfigFile { id = restic.ID{} } else { @@ -532,6 +538,25 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by return id, nil } +func (r *Repository) verifyUnpacked(buf []byte, t restic.FileType, expected []byte) error { + nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] + plaintext, err := r.key.Open(nil, nonce, ciphertext, nil) + if err != nil { + return fmt.Errorf("decryption failed: %w", err) + } + if t != restic.ConfigFile { + plaintext, err = r.decompressUnpacked(plaintext) + if err != nil { + return fmt.Errorf("decompression failed: %w", err) + } + } + + if !bytes.Equal(plaintext, expected) { + return errors.New("data mismatch") + } + return nil +} + // Flush saves all remaining packs and the index func (r *Repository) Flush(ctx context.Context) error { if err := r.flushPacks(ctx); err != nil { From 2dbb18128c4923f271691ad2530cb24b60d520b5 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 17:47:48 +0100 Subject: [PATCH 094/158] repository: Allow skipping verification for tests Some tests have to explicitly create pack files with blobs that don't match their ID. For those blobs the builtin verification of the repository must be disabled. --- internal/repository/repack_test.go | 6 ++++-- internal/repository/repair_pack_test.go | 3 ++- internal/repository/repository.go | 13 +++++++++++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index 63845d5b1..c07c0a943 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -336,7 +336,8 @@ func TestRepackWrongBlob(t *testing.T) { } func testRepackWrongBlob(t *testing.T, version uint) { - repo := repository.TestRepositoryWithVersion(t, version) + // disable verification to allow adding corrupted blobs to the repository + repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoVerifyPack: true}) seed := time.Now().UnixNano() rand.Seed(seed) @@ -361,7 +362,8 @@ func TestRepackBlobFallback(t *testing.T) { } func testRepackBlobFallback(t *testing.T, version uint) { - repo := repository.TestRepositoryWithVersion(t, version) + // disable verification to allow adding corrupted blobs to the repository + repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoVerifyPack: true}) seed := time.Now().UnixNano() rand.Seed(seed) diff --git a/internal/repository/repair_pack_test.go b/internal/repository/repair_pack_test.go index 3f9477945..c9b0badfc 100644 --- a/internal/repository/repair_pack_test.go +++ b/internal/repository/repair_pack_test.go @@ -102,7 +102,8 @@ func testRepairBrokenPack(t *testing.T, version uint) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - repo := repository.TestRepositoryWithVersion(t, version) + // disable verification to allow adding corrupted blobs to the repository + repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoVerifyPack: true}) seed := time.Now().UnixNano() rand.Seed(seed) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 2584f42c7..706e84876 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -59,8 +59,9 @@ type Repository struct { } type Options struct { - Compression CompressionMode - PackSize uint + Compression CompressionMode + PackSize uint + NoVerifyPack bool } // CompressionMode configures if data should be compressed. @@ -444,6 +445,10 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data } func (r *Repository) verifyCiphertext(buf []byte, uncompressedLength int, id restic.ID) error { + if r.opts.NoVerifyPack { + return nil + } + nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] plaintext, err := r.key.Open(nil, nonce, ciphertext, nil) if err != nil { @@ -539,6 +544,10 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf [] } func (r *Repository) verifyUnpacked(buf []byte, t restic.FileType, expected []byte) error { + if r.opts.NoVerifyPack { + return nil + } + nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] plaintext, err := r.key.Open(nil, nonce, ciphertext, nil) if err != nil { From 193140525c1358096216c6198866c5306015cd3e Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 18:13:34 +0100 Subject: [PATCH 095/158] repository: test verification of blobs/unpacked data --- .../repository/repository_internal_test.go | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/internal/repository/repository_internal_test.go b/internal/repository/repository_internal_test.go index eed99c7e0..0c7115bc9 100644 --- a/internal/repository/repository_internal_test.go +++ b/internal/repository/repository_internal_test.go @@ -351,3 +351,101 @@ func testStreamPack(t *testing.T, version uint) { } }) } + +func TestBlobVerification(t *testing.T) { + repo := TestRepository(t).(*Repository) + + type DamageType string + const ( + damageData DamageType = "data" + damageCompressed DamageType = "compressed" + damageCiphertext DamageType = "ciphertext" + ) + + for _, test := range []struct { + damage DamageType + msg string + }{ + {"", ""}, + {damageData, "hash mismatch"}, + {damageCompressed, "decompression failed"}, + {damageCiphertext, "ciphertext verification failed"}, + } { + plaintext := rtest.Random(800, 1234) + id := restic.Hash(plaintext) + if test.damage == damageData { + plaintext[42] ^= 0x42 + } + + uncompressedLength := uint(len(plaintext)) + plaintext = repo.getZstdEncoder().EncodeAll(plaintext, nil) + + if test.damage == damageCompressed { + plaintext = plaintext[:len(plaintext)-8] + } + + nonce := crypto.NewRandomNonce() + ciphertext := append([]byte{}, nonce...) + ciphertext = repo.Key().Seal(ciphertext, nonce, plaintext, nil) + + if test.damage == damageCiphertext { + ciphertext[42] ^= 0x42 + } + + err := repo.verifyCiphertext(ciphertext, int(uncompressedLength), id) + if test.msg == "" { + rtest.Assert(t, err == nil, "expected no error, got %v", err) + } else { + rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err) + } + } +} + +func TestUnpackedVerification(t *testing.T) { + repo := TestRepository(t).(*Repository) + + type DamageType string + const ( + damageData DamageType = "data" + damageCompressed DamageType = "compressed" + damageCiphertext DamageType = "ciphertext" + ) + + for _, test := range []struct { + damage DamageType + msg string + }{ + {"", ""}, + {damageData, "data mismatch"}, + {damageCompressed, "decompression failed"}, + {damageCiphertext, "ciphertext verification failed"}, + } { + plaintext := rtest.Random(800, 1234) + orig := append([]byte{}, plaintext...) + if test.damage == damageData { + plaintext[42] ^= 0x42 + } + + compressed := []byte{2} + compressed = repo.getZstdEncoder().EncodeAll(plaintext, compressed) + + if test.damage == damageCompressed { + compressed = compressed[:len(compressed)-8] + } + + nonce := crypto.NewRandomNonce() + ciphertext := append([]byte{}, nonce...) + ciphertext = repo.Key().Seal(ciphertext, nonce, compressed, nil) + + if test.damage == damageCiphertext { + ciphertext[42] ^= 0x42 + } + + err := repo.verifyUnpacked(ciphertext, restic.IndexFile, orig) + if test.msg == "" { + rtest.Assert(t, err == nil, "expected no error, got %v", err) + } else { + rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err) + } + } +} From 66e8971659bdd84a0363799e2206ef310fbc8fc2 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 18:17:18 +0100 Subject: [PATCH 096/158] Make --no-verify-pack globally available Verifying all blobs before upload comes with a notable performance impact. Allow users to skip it if necessary. --- cmd/restic/global.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 6def9bf83..49cb894fa 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -67,6 +67,7 @@ type GlobalOptions struct { CleanupCache bool Compression repository.CompressionMode PackSize uint + NoVerifyPack bool backend.TransportOptions limiter.Limits @@ -139,6 +140,7 @@ func init() { f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)") f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)") + f.BoolVar(&globalOptions.NoVerifyPack, "no-verify-pack", false, "skip verification of data before upload") f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)") f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)") f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)") @@ -453,8 +455,9 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi } s, err := repository.New(be, repository.Options{ - Compression: opts.Compression, - PackSize: opts.PackSize * 1024 * 1024, + Compression: opts.Compression, + PackSize: opts.PackSize * 1024 * 1024, + NoVerifyPack: opts.NoVerifyPack, }) if err != nil { return nil, errors.Fatal(err.Error()) From c97a271e89a704a0524c3a3b654e0bff90bbf5ba Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 18:42:02 +0100 Subject: [PATCH 097/158] repository: ask users to report corrupted data while saving blobs --- internal/repository/repository.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 706e84876..f8b21586a 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -425,8 +425,7 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data ciphertext = r.key.Seal(ciphertext, nonce, data, nil) if err := r.verifyCiphertext(ciphertext, uncompressedLength, id); err != nil { - // FIXME call to action - return 0, fmt.Errorf("detected data corruption while saving blob %v: %w", id, err) + return 0, fmt.Errorf("detected data corruption while saving blob %v: %w\nCorrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", id, err) } // find suitable packer and add blob @@ -522,8 +521,7 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf [] ciphertext = r.key.Seal(ciphertext, nonce, p, nil) if err := r.verifyUnpacked(ciphertext, t, buf); err != nil { - // FIXME call to action - return restic.ID{}, fmt.Errorf("detected data corruption while saving file of type %v: %w", t, err) + return restic.ID{}, fmt.Errorf("detected data corruption while saving file of type %v: %w\nCorrupted data is either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", t, err) } if t == restic.ConfigFile { From c32e5e2abba39cdff12ad1ede34078b0c6598d30 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 4 Feb 2024 11:58:29 +0100 Subject: [PATCH 098/158] pack: verify integrity of pack file header --- internal/pack/pack.go | 43 +++++++++++++++------ internal/pack/pack_internal_test.go | 58 +++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 11 deletions(-) diff --git a/internal/pack/pack.go b/internal/pack/pack.go index 211af7bfb..04042744e 100644 --- a/internal/pack/pack.go +++ b/internal/pack/pack.go @@ -1,6 +1,7 @@ package pack import ( + "bytes" "context" "encoding/binary" "fmt" @@ -74,7 +75,7 @@ func (p *Packer) Finalize() error { p.m.Lock() defer p.m.Unlock() - header, err := p.makeHeader() + header, err := makeHeader(p.blobs) if err != nil { return err } @@ -83,6 +84,11 @@ func (p *Packer) Finalize() error { nonce := crypto.NewRandomNonce() encryptedHeader = append(encryptedHeader, nonce...) encryptedHeader = p.k.Seal(encryptedHeader, nonce, header, nil) + encryptedHeader = binary.LittleEndian.AppendUint32(encryptedHeader, uint32(len(encryptedHeader))) + + if err := verifyHeader(p.k, encryptedHeader, p.blobs); err != nil { + return fmt.Errorf("detected data corruption while writing pack-file header: %w\nCorrupted data is either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", err) + } // append the header n, err := p.wr.Write(encryptedHeader) @@ -90,18 +96,33 @@ func (p *Packer) Finalize() error { return errors.Wrap(err, "Write") } - hdrBytes := len(encryptedHeader) - if n != hdrBytes { + if n != len(encryptedHeader) { return errors.New("wrong number of bytes written") } + p.bytes += uint(len(encryptedHeader)) - // write length - err = binary.Write(p.wr, binary.LittleEndian, uint32(hdrBytes)) + return nil +} + +func verifyHeader(k *crypto.Key, header []byte, expected []restic.Blob) error { + // do not offer a way to skip the pack header verification, as pack headers are usually small enough + // to not result in a significant performance impact + + decoded, hdrSize, err := List(k, bytes.NewReader(header), int64(len(header))) if err != nil { - return errors.Wrap(err, "binary.Write") + return fmt.Errorf("header decoding failed: %w", err) + } + if hdrSize != uint32(len(header)) { + return fmt.Errorf("unexpected header size %v instead of %v", hdrSize, len(header)) + } + if len(decoded) != len(expected) { + return fmt.Errorf("pack header size mismatch") + } + for i := 0; i < len(decoded); i++ { + if decoded[i] != expected[i] { + return fmt.Errorf("pack header entry mismatch got %v instead of %v", decoded[i], expected[i]) + } } - p.bytes += uint(hdrBytes + binary.Size(uint32(0))) - return nil } @@ -111,10 +132,10 @@ func (p *Packer) HeaderOverhead() int { } // makeHeader constructs the header for p. -func (p *Packer) makeHeader() ([]byte, error) { - buf := make([]byte, 0, len(p.blobs)*int(entrySize)) +func makeHeader(blobs []restic.Blob) ([]byte, error) { + buf := make([]byte, 0, len(blobs)*int(entrySize)) - for _, b := range p.blobs { + for _, b := range blobs { switch { case b.Type == restic.DataBlob && b.UncompressedLength == 0: buf = append(buf, 0) diff --git a/internal/pack/pack_internal_test.go b/internal/pack/pack_internal_test.go index c1a4867ea..2e7400ad0 100644 --- a/internal/pack/pack_internal_test.go +++ b/internal/pack/pack_internal_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "io" + "strings" "testing" "github.com/restic/restic/internal/crypto" @@ -177,3 +178,60 @@ func TestReadRecords(t *testing.T) { } } } + +func TestUnpackedVerification(t *testing.T) { + // create random keys + k := crypto.NewRandomKey() + blobs := []restic.Blob{ + { + BlobHandle: restic.NewRandomBlobHandle(), + Length: 42, + Offset: 0, + UncompressedLength: 2 * 42, + }, + } + + type DamageType string + const ( + damageData DamageType = "data" + damageCiphertext DamageType = "ciphertext" + damageLength DamageType = "length" + ) + + for _, test := range []struct { + damage DamageType + msg string + }{ + {"", ""}, + {damageData, "pack header entry mismatch"}, + {damageCiphertext, "ciphertext verification failed"}, + {damageLength, "header decoding failed"}, + } { + header, err := makeHeader(blobs) + rtest.OK(t, err) + + if test.damage == damageData { + header[8] ^= 0x42 + } + + encryptedHeader := make([]byte, 0, crypto.CiphertextLength(len(header))) + nonce := crypto.NewRandomNonce() + encryptedHeader = append(encryptedHeader, nonce...) + encryptedHeader = k.Seal(encryptedHeader, nonce, header, nil) + encryptedHeader = binary.LittleEndian.AppendUint32(encryptedHeader, uint32(len(encryptedHeader))) + + if test.damage == damageCiphertext { + encryptedHeader[8] ^= 0x42 + } + if test.damage == damageLength { + encryptedHeader[len(encryptedHeader)-1] ^= 0x42 + } + + err = verifyHeader(k, encryptedHeader, blobs) + if test.msg == "" { + rtest.Assert(t, err == nil, "expected no error, got %v", err) + } else { + rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err) + } + } +} From 7d31180fe65f0b4a0442673c0e6318c593263b1f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 4 Feb 2024 15:48:11 +0100 Subject: [PATCH 099/158] add data verification changelog entry --- changelog/unreleased/issue-4529 | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 changelog/unreleased/issue-4529 diff --git a/changelog/unreleased/issue-4529 b/changelog/unreleased/issue-4529 new file mode 100644 index 000000000..2e8bbbed7 --- /dev/null +++ b/changelog/unreleased/issue-4529 @@ -0,0 +1,14 @@ +Enhancement: Verify data integrity before upload + +Hardware issues or a bug in restic could cause restic to create corrupted files +that were then uploaded to the repository. Detecting such corruption usually +required explicitly running the `check --read-data` command. + +To prevent the upload of corrupted data to the repository, restic now +additionally verifies that files can be decoded and contain the correct data +beforehand. This increases the CPU usage during backups. If absolutely +necessary, you can disable the verification using the option +`--no-verify-pack`. + +https://github.com/restic/restic/issues/4529 +https://github.com/restic/restic/pull/4681 From 86b38a0b17f4b67080047516913def0d0bc5e0a0 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 4 Feb 2024 16:50:50 +0100 Subject: [PATCH 100/158] rename `--no-verify-pack` to `--no-extra-verify` --- changelog/unreleased/issue-4529 | 2 +- cmd/restic/global.go | 10 +++++----- internal/repository/repack_test.go | 4 ++-- internal/repository/repair_pack_test.go | 2 +- internal/repository/repository.go | 10 +++++----- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/changelog/unreleased/issue-4529 b/changelog/unreleased/issue-4529 index 2e8bbbed7..c3ec69510 100644 --- a/changelog/unreleased/issue-4529 +++ b/changelog/unreleased/issue-4529 @@ -8,7 +8,7 @@ To prevent the upload of corrupted data to the repository, restic now additionally verifies that files can be decoded and contain the correct data beforehand. This increases the CPU usage during backups. If absolutely necessary, you can disable the verification using the option -`--no-verify-pack`. +`--no-extra-verify`. https://github.com/restic/restic/issues/4529 https://github.com/restic/restic/pull/4681 diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 49cb894fa..08342435a 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -67,7 +67,7 @@ type GlobalOptions struct { CleanupCache bool Compression repository.CompressionMode PackSize uint - NoVerifyPack bool + NoExtraVerify bool backend.TransportOptions limiter.Limits @@ -140,7 +140,7 @@ func init() { f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)") f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)") - f.BoolVar(&globalOptions.NoVerifyPack, "no-verify-pack", false, "skip verification of data before upload") + f.BoolVar(&globalOptions.NoExtraVerify, "no-extra-verify", false, "skip verification of data before upload") f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)") f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)") f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)") @@ -455,9 +455,9 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi } s, err := repository.New(be, repository.Options{ - Compression: opts.Compression, - PackSize: opts.PackSize * 1024 * 1024, - NoVerifyPack: opts.NoVerifyPack, + Compression: opts.Compression, + PackSize: opts.PackSize * 1024 * 1024, + NoExtraVerify: opts.NoExtraVerify, }) if err != nil { return nil, errors.Fatal(err.Error()) diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index c07c0a943..e5e46ac2a 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -337,7 +337,7 @@ func TestRepackWrongBlob(t *testing.T) { func testRepackWrongBlob(t *testing.T, version uint) { // disable verification to allow adding corrupted blobs to the repository - repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoVerifyPack: true}) + repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true}) seed := time.Now().UnixNano() rand.Seed(seed) @@ -363,7 +363,7 @@ func TestRepackBlobFallback(t *testing.T) { func testRepackBlobFallback(t *testing.T, version uint) { // disable verification to allow adding corrupted blobs to the repository - repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoVerifyPack: true}) + repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true}) seed := time.Now().UnixNano() rand.Seed(seed) diff --git a/internal/repository/repair_pack_test.go b/internal/repository/repair_pack_test.go index c9b0badfc..b950245aa 100644 --- a/internal/repository/repair_pack_test.go +++ b/internal/repository/repair_pack_test.go @@ -103,7 +103,7 @@ func testRepairBrokenPack(t *testing.T, version uint) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { // disable verification to allow adding corrupted blobs to the repository - repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoVerifyPack: true}) + repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true}) seed := time.Now().UnixNano() rand.Seed(seed) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index f8b21586a..917b7318f 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -59,9 +59,9 @@ type Repository struct { } type Options struct { - Compression CompressionMode - PackSize uint - NoVerifyPack bool + Compression CompressionMode + PackSize uint + NoExtraVerify bool } // CompressionMode configures if data should be compressed. @@ -444,7 +444,7 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data } func (r *Repository) verifyCiphertext(buf []byte, uncompressedLength int, id restic.ID) error { - if r.opts.NoVerifyPack { + if r.opts.NoExtraVerify { return nil } @@ -542,7 +542,7 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf [] } func (r *Repository) verifyUnpacked(buf []byte, t restic.FileType, expected []byte) error { - if r.opts.NoVerifyPack { + if r.opts.NoExtraVerify { return nil } From a737fe1e478f7a06c1ae7c5e0a8569edccaf3b47 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 4 Feb 2024 17:11:49 +0100 Subject: [PATCH 101/158] add documentation for --no-extra-verify option --- doc/047_tuning_backup_parameters.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/047_tuning_backup_parameters.rst b/doc/047_tuning_backup_parameters.rst index 6ea39dc75..5773ac161 100644 --- a/doc/047_tuning_backup_parameters.rst +++ b/doc/047_tuning_backup_parameters.rst @@ -60,6 +60,17 @@ only applied for the single run of restic. The option can also be set via the en variable ``RESTIC_COMPRESSION``. +Data Verification +================= + +To prevent the upload of corrupted data to the repository, restic verifies that files can +be decoded and contain the correct data beforehand. This increases the CPU usage during +backups. If necessary, you can disable this verification using the option ``--no-extra-verify``. +However, in this case you should verify the repository integrity more actively using +``restic check --read-data``. Otherwise, data corruption due to hardware issues or software +bugs might go unnoticed. + + File Read Concurrency ===================== From 219d8e3c18e8b1e7a861dfe498debfca7f99a0d7 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 18:52:51 +0100 Subject: [PATCH 102/158] add changelog draft for data corruption on max compression --- changelog/unreleased/issue-4677 | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 changelog/unreleased/issue-4677 diff --git a/changelog/unreleased/issue-4677 b/changelog/unreleased/issue-4677 new file mode 100644 index 000000000..8fa6cf65b --- /dev/null +++ b/changelog/unreleased/issue-4677 @@ -0,0 +1,19 @@ +Bugfix: Downgrade zstd library to fix rare data corruption at max. compression + +In restic 0.16.3, backups where the compression level was set to `max` (using +`--compression max`) could in rare and very specific circumstances result in +data corruption due to a bug in the library used for compressing data. Restic +0.16.1 and 0.16.2 were not affected. + +Restic now uses the previous version of the library used to compress data, the +same version used by restic 0.16.2. Please note that the `auto` compression +level (which restic uses by default) was never affected, and even if you used +`max` compression, chances of being affected by this issue are small. + +To check a repository for any corruption, run `restic check --read-data`. This +will download and verify the whole repository and can be used at any time to +completely verify the integrity of a repository. If the `check` command detects +anomalies, follow the suggested steps. + +https://github.com/restic/restic/issues/4677 +https://github.com/restic/restic/pull/4679 From 5957417b1f85e0921dae56e88ef68f3d5ce88e9f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 4 Feb 2024 18:09:32 +0100 Subject: [PATCH 103/158] Apply changelog entry / documentation improvements from review --- changelog/unreleased/issue-4529 | 22 +++++++++++++--------- cmd/restic/global.go | 2 +- doc/047_tuning_backup_parameters.rst | 15 +++++++++------ internal/pack/pack.go | 3 ++- internal/repository/repository.go | 6 ++++-- 5 files changed, 29 insertions(+), 19 deletions(-) diff --git a/changelog/unreleased/issue-4529 b/changelog/unreleased/issue-4529 index c3ec69510..fed726d2d 100644 --- a/changelog/unreleased/issue-4529 +++ b/changelog/unreleased/issue-4529 @@ -1,14 +1,18 @@ -Enhancement: Verify data integrity before upload +Enhancement: Add extra verification of data integrity before upload -Hardware issues or a bug in restic could cause restic to create corrupted files -that were then uploaded to the repository. Detecting such corruption usually -required explicitly running the `check --read-data` command. +Hardware issues, or a bug in restic or its dependencies, could previously cause +corruption in the files restic created and stored in the repository. Detecting +such corruption previously required explicitly running the `check --read-data` +or `check --read-data-subset` commands. -To prevent the upload of corrupted data to the repository, restic now -additionally verifies that files can be decoded and contain the correct data -beforehand. This increases the CPU usage during backups. If absolutely -necessary, you can disable the verification using the option -`--no-extra-verify`. +To further ensure data integrity, even in the case of hardware issues or +software bugs, restic now performs additional verification of the files about +to be uploaded to the repository. + +These extra checks will increase CPU usage during backups. They can therefore, +if absolutely necessary, be disabled using the `--no-extra-verify` global +option. Please note that this should be combined with more active checking +using the previously mentioned check commands. https://github.com/restic/restic/issues/4529 https://github.com/restic/restic/pull/4681 diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 08342435a..da01aa732 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -140,7 +140,7 @@ func init() { f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)") f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)") - f.BoolVar(&globalOptions.NoExtraVerify, "no-extra-verify", false, "skip verification of data before upload") + f.BoolVar(&globalOptions.NoExtraVerify, "no-extra-verify", false, "skip additional verification of data before upload (see documentation)") f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)") f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)") f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)") diff --git a/doc/047_tuning_backup_parameters.rst b/doc/047_tuning_backup_parameters.rst index 5773ac161..d8fb2c9b6 100644 --- a/doc/047_tuning_backup_parameters.rst +++ b/doc/047_tuning_backup_parameters.rst @@ -63,12 +63,15 @@ variable ``RESTIC_COMPRESSION``. Data Verification ================= -To prevent the upload of corrupted data to the repository, restic verifies that files can -be decoded and contain the correct data beforehand. This increases the CPU usage during -backups. If necessary, you can disable this verification using the option ``--no-extra-verify``. -However, in this case you should verify the repository integrity more actively using -``restic check --read-data``. Otherwise, data corruption due to hardware issues or software -bugs might go unnoticed. +To prevent the upload of corrupted data to the repository, which can happen due +to hardware issues or software bugs, restic verifies that generated files can +be decoded and contain the correct data beforehand. This increases the CPU usage +during backups. If necessary, you can disable this verification using the +``--no-extra-verify`` option of the ``backup`` command. However, in this case +you should verify the repository integrity more actively using +``restic check --read-data`` (or the similar ``--read-data-subset`` option). +Otherwise, data corruption due to hardware issues or software bugs might go +unnoticed. File Read Concurrency diff --git a/internal/pack/pack.go b/internal/pack/pack.go index 04042744e..cd118ab03 100644 --- a/internal/pack/pack.go +++ b/internal/pack/pack.go @@ -87,7 +87,8 @@ func (p *Packer) Finalize() error { encryptedHeader = binary.LittleEndian.AppendUint32(encryptedHeader, uint32(len(encryptedHeader))) if err := verifyHeader(p.k, encryptedHeader, p.blobs); err != nil { - return fmt.Errorf("detected data corruption while writing pack-file header: %w\nCorrupted data is either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", err) + //nolint:revive // ignore linter warnings about error message spelling + return fmt.Errorf("Detected data corruption while writing pack-file header: %w\nCorrupted data is either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", err) } // append the header diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 917b7318f..a20f71ab1 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -425,7 +425,8 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data ciphertext = r.key.Seal(ciphertext, nonce, data, nil) if err := r.verifyCiphertext(ciphertext, uncompressedLength, id); err != nil { - return 0, fmt.Errorf("detected data corruption while saving blob %v: %w\nCorrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", id, err) + //nolint:revive // ignore linter warnings about error message spelling + return 0, fmt.Errorf("Detected data corruption while saving blob %v: %w\nCorrupted blobs are either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", id, err) } // find suitable packer and add blob @@ -521,7 +522,8 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf [] ciphertext = r.key.Seal(ciphertext, nonce, p, nil) if err := r.verifyUnpacked(ciphertext, t, buf); err != nil { - return restic.ID{}, fmt.Errorf("detected data corruption while saving file of type %v: %w\nCorrupted data is either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", t, err) + //nolint:revive // ignore linter warnings about error message spelling + return restic.ID{}, fmt.Errorf("Detected data corruption while saving file of type %v: %w\nCorrupted data is either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", t, err) } if t == restic.ConfigFile { From 0164f5310d627b2e920f7427c265483634b020da Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 2 Feb 2024 20:10:29 +0100 Subject: [PATCH 104/158] Downgrade klauspost/compress to fix data corruption at max. compression --- go.mod | 2 ++ go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index f07fc10ee..970770a90 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,8 @@ require ( google.golang.org/api v0.149.0 ) +replace github.com/klauspost/compress => github.com/klauspost/compress v1.17.2 + require ( cloud.google.com/go v0.110.9 // indirect cloud.google.com/go/compute v1.23.1 // indirect diff --git a/go.sum b/go.sum index 01e58cccb..129eafed2 100644 --- a/go.sum +++ b/go.sum @@ -108,8 +108,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= -github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= From 29e1caf8250c1965467fb6da1f6155e953b4075d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 18:52:51 +0100 Subject: [PATCH 105/158] add changelog draft for data corruption on max compression --- changelog/unreleased/issue-4677 | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 changelog/unreleased/issue-4677 diff --git a/changelog/unreleased/issue-4677 b/changelog/unreleased/issue-4677 new file mode 100644 index 000000000..8fa6cf65b --- /dev/null +++ b/changelog/unreleased/issue-4677 @@ -0,0 +1,19 @@ +Bugfix: Downgrade zstd library to fix rare data corruption at max. compression + +In restic 0.16.3, backups where the compression level was set to `max` (using +`--compression max`) could in rare and very specific circumstances result in +data corruption due to a bug in the library used for compressing data. Restic +0.16.1 and 0.16.2 were not affected. + +Restic now uses the previous version of the library used to compress data, the +same version used by restic 0.16.2. Please note that the `auto` compression +level (which restic uses by default) was never affected, and even if you used +`max` compression, chances of being affected by this issue are small. + +To check a repository for any corruption, run `restic check --read-data`. This +will download and verify the whole repository and can be used at any time to +completely verify the integrity of a repository. If the `check` command detects +anomalies, follow the suggested steps. + +https://github.com/restic/restic/issues/4677 +https://github.com/restic/restic/pull/4679 From 0ea62b5ac6804a425ebbb9dfbfe65c296836e36b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 17:47:36 +0100 Subject: [PATCH 106/158] repository: make repo.Options configurable for test repos --- internal/archiver/archiver_test.go | 2 +- internal/migrations/upgrade_repo_v2_test.go | 2 +- internal/repository/fuzz_test.go | 3 +-- internal/repository/testing.go | 9 +++++---- internal/restic/lock_test.go | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/internal/archiver/archiver_test.go b/internal/archiver/archiver_test.go index 3c87055d8..5a9896a48 100644 --- a/internal/archiver/archiver_test.go +++ b/internal/archiver/archiver_test.go @@ -1879,7 +1879,7 @@ func TestArchiverContextCanceled(t *testing.T) { }) // Ensure that the archiver itself reports the canceled context and not just the backend - repo := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0) + repo := repository.TestRepositoryWithBackend(t, &noCancelBackend{mem.New()}, 0, repository.Options{}) back := restictest.Chdir(t, tempdir) defer back() diff --git a/internal/migrations/upgrade_repo_v2_test.go b/internal/migrations/upgrade_repo_v2_test.go index 96fc7788e..7f251de93 100644 --- a/internal/migrations/upgrade_repo_v2_test.go +++ b/internal/migrations/upgrade_repo_v2_test.go @@ -69,7 +69,7 @@ func TestUpgradeRepoV2Failure(t *testing.T) { Backend: be, } - repo := repository.TestRepositoryWithBackend(t, be, 1) + repo := repository.TestRepositoryWithBackend(t, be, 1, repository.Options{}) if repo.Config().Version != 1 { t.Fatal("test repo has wrong version") } diff --git a/internal/repository/fuzz_test.go b/internal/repository/fuzz_test.go index b4036288c..80372f8e0 100644 --- a/internal/repository/fuzz_test.go +++ b/internal/repository/fuzz_test.go @@ -4,7 +4,6 @@ import ( "context" "testing" - "github.com/restic/restic/internal/backend/mem" "github.com/restic/restic/internal/restic" "golang.org/x/sync/errgroup" ) @@ -19,7 +18,7 @@ func FuzzSaveLoadBlob(f *testing.F) { } id := restic.Hash(blob) - repo := TestRepositoryWithBackend(t, mem.New(), 2) + repo := TestRepositoryWithVersion(t, 2) var wg errgroup.Group repo.StartPackUploader(context.TODO(), &wg) diff --git a/internal/repository/testing.go b/internal/repository/testing.go index 4936cc368..9bdd65901 100644 --- a/internal/repository/testing.go +++ b/internal/repository/testing.go @@ -43,7 +43,7 @@ const TestChunkerPol = chunker.Pol(0x3DA3358B4DC173) // TestRepositoryWithBackend returns a repository initialized with a test // password. If be is nil, an in-memory backend is used. A constant polynomial // is used for the chunker and low-security test parameters. -func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint) restic.Repository { +func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint, opts Options) restic.Repository { t.Helper() TestUseLowSecurityKDFParameters(t) restic.TestDisableCheckPolynomial(t) @@ -52,7 +52,7 @@ func TestRepositoryWithBackend(t testing.TB, be restic.Backend, version uint) re be = TestBackend(t) } - repo, err := New(be, Options{}) + repo, err := New(be, opts) if err != nil { t.Fatalf("TestRepository(): new repo failed: %v", err) } @@ -78,6 +78,7 @@ func TestRepository(t testing.TB) restic.Repository { func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository { t.Helper() dir := os.Getenv("RESTIC_TEST_REPO") + opts := Options{} if dir != "" { _, err := os.Stat(dir) if err != nil { @@ -85,7 +86,7 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository { if err != nil { t.Fatalf("error creating local backend at %v: %v", dir, err) } - return TestRepositoryWithBackend(t, be, version) + return TestRepositoryWithBackend(t, be, version, opts) } if err == nil { @@ -93,7 +94,7 @@ func TestRepositoryWithVersion(t testing.TB, version uint) restic.Repository { } } - return TestRepositoryWithBackend(t, nil, version) + return TestRepositoryWithBackend(t, nil, version, opts) } // TestOpenLocal opens a local repository. diff --git a/internal/restic/lock_test.go b/internal/restic/lock_test.go index f3c405c9c..2eb22be1b 100644 --- a/internal/restic/lock_test.go +++ b/internal/restic/lock_test.go @@ -65,7 +65,7 @@ func (be *failLockLoadingBackend) Load(ctx context.Context, h restic.Handle, len func TestMultipleLockFailure(t *testing.T) { be := &failLockLoadingBackend{Backend: mem.New()} - repo := repository.TestRepositoryWithBackend(t, be, 0) + repo := repository.TestRepositoryWithBackend(t, be, 0, repository.Options{}) restic.TestSetLockTimeout(t, 5*time.Millisecond) lock1, err := restic.NewLock(context.TODO(), repo) From 2f30c940b2f6ca645f87a06a841ff137e3153afe Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Fri, 2 Feb 2024 21:15:39 +0100 Subject: [PATCH 107/158] backup: verify blobs before upload This only covers the blobs themselves, the pack header is not verified so far. Unpacked files are also not covered by the integrity check. --- internal/repository/repository.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 0b50382b8..7855438c9 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -423,6 +423,11 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data // encrypt blob ciphertext = r.key.Seal(ciphertext, nonce, data, nil) + if err := r.verifyCiphertext(ciphertext, uncompressedLength, id); err != nil { + // FIXME call to action + return 0, fmt.Errorf("detected data corruption while saving blob %v: %w", id, err) + } + // find suitable packer and add blob var pm *packerManager @@ -438,6 +443,27 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data return pm.SaveBlob(ctx, t, id, ciphertext, uncompressedLength) } +func (r *Repository) verifyCiphertext(buf []byte, uncompressedLength int, id restic.ID) error { + nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] + plaintext, err := r.key.Open(nil, nonce, ciphertext, nil) + if err != nil { + return fmt.Errorf("decryption failed: %w", err) + } + if uncompressedLength != 0 { + // DecodeAll will allocate a slice if it is not large enough since it + // knows the decompressed size (because we're using EncodeAll) + plaintext, err = r.getZstdDecoder().DecodeAll(plaintext, nil) + if err != nil { + return fmt.Errorf("decompression failed: %w", err) + } + } + if !restic.Hash(plaintext).Equal(id) { + return errors.New("hash mismatch") + } + + return nil +} + func (r *Repository) compressUnpacked(p []byte) ([]byte, error) { // compression is only available starting from version 2 if r.cfg.Version < 2 { From cb85fb46ddf15d8c6d149d23cadcaab5e7164786 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 17:30:58 +0100 Subject: [PATCH 108/158] backup: verify unpacked files before upload --- internal/repository/repository.go | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 7855438c9..4fa7e487c 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -500,7 +500,8 @@ func (r *Repository) decompressUnpacked(p []byte) ([]byte, error) { // SaveUnpacked encrypts data and stores it in the backend. Returned is the // storage hash. -func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []byte) (id restic.ID, err error) { +func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf []byte) (id restic.ID, err error) { + p := buf if t != restic.ConfigFile { p, err = r.compressUnpacked(p) if err != nil { @@ -515,6 +516,11 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by ciphertext = r.key.Seal(ciphertext, nonce, p, nil) + if err := r.verifyUnpacked(ciphertext, t, buf); err != nil { + // FIXME call to action + return restic.ID{}, fmt.Errorf("detected data corruption while saving file of type %v: %w", t, err) + } + if t == restic.ConfigFile { id = restic.ID{} } else { @@ -532,6 +538,25 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, p []by return id, nil } +func (r *Repository) verifyUnpacked(buf []byte, t restic.FileType, expected []byte) error { + nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] + plaintext, err := r.key.Open(nil, nonce, ciphertext, nil) + if err != nil { + return fmt.Errorf("decryption failed: %w", err) + } + if t != restic.ConfigFile { + plaintext, err = r.decompressUnpacked(plaintext) + if err != nil { + return fmt.Errorf("decompression failed: %w", err) + } + } + + if !bytes.Equal(plaintext, expected) { + return errors.New("data mismatch") + } + return nil +} + // Flush saves all remaining packs and the index func (r *Repository) Flush(ctx context.Context) error { if err := r.flushPacks(ctx); err != nil { From 885431ec2b64bae061246ef8d0fab19322e29fe3 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 17:47:48 +0100 Subject: [PATCH 109/158] repository: Allow skipping verification for tests Some tests have to explicitly create pack files with blobs that don't match their ID. For those blobs the builtin verification of the repository must be disabled. --- internal/repository/repack_test.go | 6 ++++-- internal/repository/repository.go | 13 +++++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index c8570a9d4..00567aad9 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -346,7 +346,8 @@ func TestRepackWrongBlob(t *testing.T) { } func testRepackWrongBlob(t *testing.T, version uint) { - repo := repository.TestRepositoryWithVersion(t, version) + // disable verification to allow adding corrupted blobs to the repository + repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoVerifyPack: true}) seed := time.Now().UnixNano() rand.Seed(seed) @@ -371,7 +372,8 @@ func TestRepackBlobFallback(t *testing.T) { } func testRepackBlobFallback(t *testing.T, version uint) { - repo := repository.TestRepositoryWithVersion(t, version) + // disable verification to allow adding corrupted blobs to the repository + repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoVerifyPack: true}) seed := time.Now().UnixNano() rand.Seed(seed) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 4fa7e487c..c7c36a74a 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -59,8 +59,9 @@ type Repository struct { } type Options struct { - Compression CompressionMode - PackSize uint + Compression CompressionMode + PackSize uint + NoVerifyPack bool } // CompressionMode configures if data should be compressed. @@ -444,6 +445,10 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data } func (r *Repository) verifyCiphertext(buf []byte, uncompressedLength int, id restic.ID) error { + if r.opts.NoVerifyPack { + return nil + } + nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] plaintext, err := r.key.Open(nil, nonce, ciphertext, nil) if err != nil { @@ -539,6 +544,10 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf [] } func (r *Repository) verifyUnpacked(buf []byte, t restic.FileType, expected []byte) error { + if r.opts.NoVerifyPack { + return nil + } + nonce, ciphertext := buf[:r.key.NonceSize()], buf[r.key.NonceSize():] plaintext, err := r.key.Open(nil, nonce, ciphertext, nil) if err != nil { From 8ef5425351944a1d52fa76eb510cbd619c10ea8c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 18:13:34 +0100 Subject: [PATCH 110/158] repository: test verification of blobs/unpacked data --- .../repository/repository_internal_test.go | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/internal/repository/repository_internal_test.go b/internal/repository/repository_internal_test.go index e5ab6e5b7..2a9976ace 100644 --- a/internal/repository/repository_internal_test.go +++ b/internal/repository/repository_internal_test.go @@ -3,8 +3,10 @@ package repository import ( "math/rand" "sort" + "strings" "testing" + "github.com/restic/restic/internal/crypto" "github.com/restic/restic/internal/restic" rtest "github.com/restic/restic/internal/test" ) @@ -72,3 +74,101 @@ func BenchmarkSortCachedPacksFirst(b *testing.B) { sortCachedPacksFirst(cache, cpy[:]) } } + +func TestBlobVerification(t *testing.T) { + repo := TestRepository(t).(*Repository) + + type DamageType string + const ( + damageData DamageType = "data" + damageCompressed DamageType = "compressed" + damageCiphertext DamageType = "ciphertext" + ) + + for _, test := range []struct { + damage DamageType + msg string + }{ + {"", ""}, + {damageData, "hash mismatch"}, + {damageCompressed, "decompression failed"}, + {damageCiphertext, "ciphertext verification failed"}, + } { + plaintext := rtest.Random(800, 1234) + id := restic.Hash(plaintext) + if test.damage == damageData { + plaintext[42] ^= 0x42 + } + + uncompressedLength := uint(len(plaintext)) + plaintext = repo.getZstdEncoder().EncodeAll(plaintext, nil) + + if test.damage == damageCompressed { + plaintext = plaintext[:len(plaintext)-8] + } + + nonce := crypto.NewRandomNonce() + ciphertext := append([]byte{}, nonce...) + ciphertext = repo.Key().Seal(ciphertext, nonce, plaintext, nil) + + if test.damage == damageCiphertext { + ciphertext[42] ^= 0x42 + } + + err := repo.verifyCiphertext(ciphertext, int(uncompressedLength), id) + if test.msg == "" { + rtest.Assert(t, err == nil, "expected no error, got %v", err) + } else { + rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err) + } + } +} + +func TestUnpackedVerification(t *testing.T) { + repo := TestRepository(t).(*Repository) + + type DamageType string + const ( + damageData DamageType = "data" + damageCompressed DamageType = "compressed" + damageCiphertext DamageType = "ciphertext" + ) + + for _, test := range []struct { + damage DamageType + msg string + }{ + {"", ""}, + {damageData, "data mismatch"}, + {damageCompressed, "decompression failed"}, + {damageCiphertext, "ciphertext verification failed"}, + } { + plaintext := rtest.Random(800, 1234) + orig := append([]byte{}, plaintext...) + if test.damage == damageData { + plaintext[42] ^= 0x42 + } + + compressed := []byte{2} + compressed = repo.getZstdEncoder().EncodeAll(plaintext, compressed) + + if test.damage == damageCompressed { + compressed = compressed[:len(compressed)-8] + } + + nonce := crypto.NewRandomNonce() + ciphertext := append([]byte{}, nonce...) + ciphertext = repo.Key().Seal(ciphertext, nonce, compressed, nil) + + if test.damage == damageCiphertext { + ciphertext[42] ^= 0x42 + } + + err := repo.verifyUnpacked(ciphertext, restic.IndexFile, orig) + if test.msg == "" { + rtest.Assert(t, err == nil, "expected no error, got %v", err) + } else { + rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err) + } + } +} From dc11d012bb5d09dd0ae246c2e5c911b6808fde00 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 18:17:18 +0100 Subject: [PATCH 111/158] Make --no-verify-pack globally available Verifying all blobs before upload comes with a notable performance impact. Allow users to skip it if necessary. --- cmd/restic/global.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 9e3a3a7a9..5bdf03bdc 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -67,6 +67,7 @@ type GlobalOptions struct { CleanupCache bool Compression repository.CompressionMode PackSize uint + NoVerifyPack bool backend.TransportOptions limiter.Limits @@ -139,6 +140,7 @@ func init() { f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)") f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)") + f.BoolVar(&globalOptions.NoVerifyPack, "no-verify-pack", false, "skip verification of data before upload") f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)") f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)") f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)") @@ -453,8 +455,9 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi } s, err := repository.New(be, repository.Options{ - Compression: opts.Compression, - PackSize: opts.PackSize * 1024 * 1024, + Compression: opts.Compression, + PackSize: opts.PackSize * 1024 * 1024, + NoVerifyPack: opts.NoVerifyPack, }) if err != nil { return nil, errors.Fatal(err.Error()) From d8916bc3d9987a376f9748da7a306bdad8e79ed4 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 3 Feb 2024 18:42:02 +0100 Subject: [PATCH 112/158] repository: ask users to report corrupted data while saving blobs --- internal/repository/repository.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index c7c36a74a..1e253c24a 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -425,8 +425,7 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data ciphertext = r.key.Seal(ciphertext, nonce, data, nil) if err := r.verifyCiphertext(ciphertext, uncompressedLength, id); err != nil { - // FIXME call to action - return 0, fmt.Errorf("detected data corruption while saving blob %v: %w", id, err) + return 0, fmt.Errorf("detected data corruption while saving blob %v: %w\nCorrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", id, err) } // find suitable packer and add blob @@ -522,8 +521,7 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf [] ciphertext = r.key.Seal(ciphertext, nonce, p, nil) if err := r.verifyUnpacked(ciphertext, t, buf); err != nil { - // FIXME call to action - return restic.ID{}, fmt.Errorf("detected data corruption while saving file of type %v: %w", t, err) + return restic.ID{}, fmt.Errorf("detected data corruption while saving file of type %v: %w\nCorrupted data is either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", t, err) } if t == restic.ConfigFile { From 75e72d826ce9b117430f1d594a24d4954691f60c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 4 Feb 2024 11:58:29 +0100 Subject: [PATCH 113/158] pack: verify integrity of pack file header --- internal/pack/pack.go | 43 +++++++++++++++------ internal/pack/pack_internal_test.go | 58 +++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 11 deletions(-) diff --git a/internal/pack/pack.go b/internal/pack/pack.go index 34ad9d071..34e87f1f9 100644 --- a/internal/pack/pack.go +++ b/internal/pack/pack.go @@ -1,6 +1,7 @@ package pack import ( + "bytes" "context" "encoding/binary" "fmt" @@ -74,7 +75,7 @@ func (p *Packer) Finalize() error { p.m.Lock() defer p.m.Unlock() - header, err := p.makeHeader() + header, err := makeHeader(p.blobs) if err != nil { return err } @@ -83,6 +84,11 @@ func (p *Packer) Finalize() error { nonce := crypto.NewRandomNonce() encryptedHeader = append(encryptedHeader, nonce...) encryptedHeader = p.k.Seal(encryptedHeader, nonce, header, nil) + encryptedHeader = binary.LittleEndian.AppendUint32(encryptedHeader, uint32(len(encryptedHeader))) + + if err := verifyHeader(p.k, encryptedHeader, p.blobs); err != nil { + return fmt.Errorf("detected data corruption while writing pack-file header: %w\nCorrupted data is either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", err) + } // append the header n, err := p.wr.Write(encryptedHeader) @@ -90,18 +96,33 @@ func (p *Packer) Finalize() error { return errors.Wrap(err, "Write") } - hdrBytes := len(encryptedHeader) - if n != hdrBytes { + if n != len(encryptedHeader) { return errors.New("wrong number of bytes written") } + p.bytes += uint(len(encryptedHeader)) - // write length - err = binary.Write(p.wr, binary.LittleEndian, uint32(hdrBytes)) + return nil +} + +func verifyHeader(k *crypto.Key, header []byte, expected []restic.Blob) error { + // do not offer a way to skip the pack header verification, as pack headers are usually small enough + // to not result in a significant performance impact + + decoded, hdrSize, err := List(k, bytes.NewReader(header), int64(len(header))) if err != nil { - return errors.Wrap(err, "binary.Write") + return fmt.Errorf("header decoding failed: %w", err) + } + if hdrSize != uint32(len(header)) { + return fmt.Errorf("unexpected header size %v instead of %v", hdrSize, len(header)) + } + if len(decoded) != len(expected) { + return fmt.Errorf("pack header size mismatch") + } + for i := 0; i < len(decoded); i++ { + if decoded[i] != expected[i] { + return fmt.Errorf("pack header entry mismatch got %v instead of %v", decoded[i], expected[i]) + } } - p.bytes += uint(hdrBytes + binary.Size(uint32(0))) - return nil } @@ -111,10 +132,10 @@ func (p *Packer) HeaderOverhead() int { } // makeHeader constructs the header for p. -func (p *Packer) makeHeader() ([]byte, error) { - buf := make([]byte, 0, len(p.blobs)*int(entrySize)) +func makeHeader(blobs []restic.Blob) ([]byte, error) { + buf := make([]byte, 0, len(blobs)*int(entrySize)) - for _, b := range p.blobs { + for _, b := range blobs { switch { case b.Type == restic.DataBlob && b.UncompressedLength == 0: buf = append(buf, 0) diff --git a/internal/pack/pack_internal_test.go b/internal/pack/pack_internal_test.go index c1a4867ea..2e7400ad0 100644 --- a/internal/pack/pack_internal_test.go +++ b/internal/pack/pack_internal_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "io" + "strings" "testing" "github.com/restic/restic/internal/crypto" @@ -177,3 +178,60 @@ func TestReadRecords(t *testing.T) { } } } + +func TestUnpackedVerification(t *testing.T) { + // create random keys + k := crypto.NewRandomKey() + blobs := []restic.Blob{ + { + BlobHandle: restic.NewRandomBlobHandle(), + Length: 42, + Offset: 0, + UncompressedLength: 2 * 42, + }, + } + + type DamageType string + const ( + damageData DamageType = "data" + damageCiphertext DamageType = "ciphertext" + damageLength DamageType = "length" + ) + + for _, test := range []struct { + damage DamageType + msg string + }{ + {"", ""}, + {damageData, "pack header entry mismatch"}, + {damageCiphertext, "ciphertext verification failed"}, + {damageLength, "header decoding failed"}, + } { + header, err := makeHeader(blobs) + rtest.OK(t, err) + + if test.damage == damageData { + header[8] ^= 0x42 + } + + encryptedHeader := make([]byte, 0, crypto.CiphertextLength(len(header))) + nonce := crypto.NewRandomNonce() + encryptedHeader = append(encryptedHeader, nonce...) + encryptedHeader = k.Seal(encryptedHeader, nonce, header, nil) + encryptedHeader = binary.LittleEndian.AppendUint32(encryptedHeader, uint32(len(encryptedHeader))) + + if test.damage == damageCiphertext { + encryptedHeader[8] ^= 0x42 + } + if test.damage == damageLength { + encryptedHeader[len(encryptedHeader)-1] ^= 0x42 + } + + err = verifyHeader(k, encryptedHeader, blobs) + if test.msg == "" { + rtest.Assert(t, err == nil, "expected no error, got %v", err) + } else { + rtest.Assert(t, strings.Contains(err.Error(), test.msg), "expected error to contain %q, got %q", test.msg, err) + } + } +} From 4589da7eb9bc9037c9dc3b85ee0ced0569aeb2bf Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 4 Feb 2024 15:48:11 +0100 Subject: [PATCH 114/158] add data verification changelog entry --- changelog/unreleased/issue-4529 | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 changelog/unreleased/issue-4529 diff --git a/changelog/unreleased/issue-4529 b/changelog/unreleased/issue-4529 new file mode 100644 index 000000000..2e8bbbed7 --- /dev/null +++ b/changelog/unreleased/issue-4529 @@ -0,0 +1,14 @@ +Enhancement: Verify data integrity before upload + +Hardware issues or a bug in restic could cause restic to create corrupted files +that were then uploaded to the repository. Detecting such corruption usually +required explicitly running the `check --read-data` command. + +To prevent the upload of corrupted data to the repository, restic now +additionally verifies that files can be decoded and contain the correct data +beforehand. This increases the CPU usage during backups. If absolutely +necessary, you can disable the verification using the option +`--no-verify-pack`. + +https://github.com/restic/restic/issues/4529 +https://github.com/restic/restic/pull/4681 From 2a0bd2b637b4b791f92f3d84c565aaaf39b9ddb2 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 4 Feb 2024 16:50:50 +0100 Subject: [PATCH 115/158] rename `--no-verify-pack` to `--no-extra-verify` --- changelog/unreleased/issue-4529 | 2 +- cmd/restic/global.go | 10 +++++----- internal/repository/repack_test.go | 4 ++-- internal/repository/repository.go | 10 +++++----- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/changelog/unreleased/issue-4529 b/changelog/unreleased/issue-4529 index 2e8bbbed7..c3ec69510 100644 --- a/changelog/unreleased/issue-4529 +++ b/changelog/unreleased/issue-4529 @@ -8,7 +8,7 @@ To prevent the upload of corrupted data to the repository, restic now additionally verifies that files can be decoded and contain the correct data beforehand. This increases the CPU usage during backups. If absolutely necessary, you can disable the verification using the option -`--no-verify-pack`. +`--no-extra-verify`. https://github.com/restic/restic/issues/4529 https://github.com/restic/restic/pull/4681 diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 5bdf03bdc..8568e41c3 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -67,7 +67,7 @@ type GlobalOptions struct { CleanupCache bool Compression repository.CompressionMode PackSize uint - NoVerifyPack bool + NoExtraVerify bool backend.TransportOptions limiter.Limits @@ -140,7 +140,7 @@ func init() { f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)") f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)") - f.BoolVar(&globalOptions.NoVerifyPack, "no-verify-pack", false, "skip verification of data before upload") + f.BoolVar(&globalOptions.NoExtraVerify, "no-extra-verify", false, "skip verification of data before upload") f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)") f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)") f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)") @@ -455,9 +455,9 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi } s, err := repository.New(be, repository.Options{ - Compression: opts.Compression, - PackSize: opts.PackSize * 1024 * 1024, - NoVerifyPack: opts.NoVerifyPack, + Compression: opts.Compression, + PackSize: opts.PackSize * 1024 * 1024, + NoExtraVerify: opts.NoExtraVerify, }) if err != nil { return nil, errors.Fatal(err.Error()) diff --git a/internal/repository/repack_test.go b/internal/repository/repack_test.go index 00567aad9..1ecbf3e1c 100644 --- a/internal/repository/repack_test.go +++ b/internal/repository/repack_test.go @@ -347,7 +347,7 @@ func TestRepackWrongBlob(t *testing.T) { func testRepackWrongBlob(t *testing.T, version uint) { // disable verification to allow adding corrupted blobs to the repository - repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoVerifyPack: true}) + repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true}) seed := time.Now().UnixNano() rand.Seed(seed) @@ -373,7 +373,7 @@ func TestRepackBlobFallback(t *testing.T) { func testRepackBlobFallback(t *testing.T, version uint) { // disable verification to allow adding corrupted blobs to the repository - repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoVerifyPack: true}) + repo := repository.TestRepositoryWithBackend(t, nil, version, repository.Options{NoExtraVerify: true}) seed := time.Now().UnixNano() rand.Seed(seed) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 1e253c24a..cea43b0d3 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -59,9 +59,9 @@ type Repository struct { } type Options struct { - Compression CompressionMode - PackSize uint - NoVerifyPack bool + Compression CompressionMode + PackSize uint + NoExtraVerify bool } // CompressionMode configures if data should be compressed. @@ -444,7 +444,7 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data } func (r *Repository) verifyCiphertext(buf []byte, uncompressedLength int, id restic.ID) error { - if r.opts.NoVerifyPack { + if r.opts.NoExtraVerify { return nil } @@ -542,7 +542,7 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf [] } func (r *Repository) verifyUnpacked(buf []byte, t restic.FileType, expected []byte) error { - if r.opts.NoVerifyPack { + if r.opts.NoExtraVerify { return nil } From 261b1455c79335152edb74cbc3626a6248d568f7 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 4 Feb 2024 17:11:49 +0100 Subject: [PATCH 116/158] add documentation for --no-extra-verify option --- doc/047_tuning_backup_parameters.rst | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/doc/047_tuning_backup_parameters.rst b/doc/047_tuning_backup_parameters.rst index 6ea39dc75..5773ac161 100644 --- a/doc/047_tuning_backup_parameters.rst +++ b/doc/047_tuning_backup_parameters.rst @@ -60,6 +60,17 @@ only applied for the single run of restic. The option can also be set via the en variable ``RESTIC_COMPRESSION``. +Data Verification +================= + +To prevent the upload of corrupted data to the repository, restic verifies that files can +be decoded and contain the correct data beforehand. This increases the CPU usage during +backups. If necessary, you can disable this verification using the option ``--no-extra-verify``. +However, in this case you should verify the repository integrity more actively using +``restic check --read-data``. Otherwise, data corruption due to hardware issues or software +bugs might go unnoticed. + + File Read Concurrency ===================== From a72c2b74f3a3ffa259e406462b4afda49726187d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 4 Feb 2024 18:09:32 +0100 Subject: [PATCH 117/158] Apply changelog entry / documentation improvements from review --- changelog/unreleased/issue-4529 | 22 +++++++++++++--------- cmd/restic/global.go | 2 +- doc/047_tuning_backup_parameters.rst | 15 +++++++++------ internal/pack/pack.go | 3 ++- internal/repository/repository.go | 6 ++++-- 5 files changed, 29 insertions(+), 19 deletions(-) diff --git a/changelog/unreleased/issue-4529 b/changelog/unreleased/issue-4529 index c3ec69510..fed726d2d 100644 --- a/changelog/unreleased/issue-4529 +++ b/changelog/unreleased/issue-4529 @@ -1,14 +1,18 @@ -Enhancement: Verify data integrity before upload +Enhancement: Add extra verification of data integrity before upload -Hardware issues or a bug in restic could cause restic to create corrupted files -that were then uploaded to the repository. Detecting such corruption usually -required explicitly running the `check --read-data` command. +Hardware issues, or a bug in restic or its dependencies, could previously cause +corruption in the files restic created and stored in the repository. Detecting +such corruption previously required explicitly running the `check --read-data` +or `check --read-data-subset` commands. -To prevent the upload of corrupted data to the repository, restic now -additionally verifies that files can be decoded and contain the correct data -beforehand. This increases the CPU usage during backups. If absolutely -necessary, you can disable the verification using the option -`--no-extra-verify`. +To further ensure data integrity, even in the case of hardware issues or +software bugs, restic now performs additional verification of the files about +to be uploaded to the repository. + +These extra checks will increase CPU usage during backups. They can therefore, +if absolutely necessary, be disabled using the `--no-extra-verify` global +option. Please note that this should be combined with more active checking +using the previously mentioned check commands. https://github.com/restic/restic/issues/4529 https://github.com/restic/restic/pull/4681 diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 8568e41c3..528c6e129 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -140,7 +140,7 @@ func init() { f.BoolVar(&globalOptions.InsecureTLS, "insecure-tls", false, "skip TLS certificate verification when connecting to the repository (insecure)") f.BoolVar(&globalOptions.CleanupCache, "cleanup-cache", false, "auto remove old cache directories") f.Var(&globalOptions.Compression, "compression", "compression mode (only available for repository format version 2), one of (auto|off|max) (default: $RESTIC_COMPRESSION)") - f.BoolVar(&globalOptions.NoExtraVerify, "no-extra-verify", false, "skip verification of data before upload") + f.BoolVar(&globalOptions.NoExtraVerify, "no-extra-verify", false, "skip additional verification of data before upload (see documentation)") f.IntVar(&globalOptions.Limits.UploadKb, "limit-upload", 0, "limits uploads to a maximum `rate` in KiB/s. (default: unlimited)") f.IntVar(&globalOptions.Limits.DownloadKb, "limit-download", 0, "limits downloads to a maximum `rate` in KiB/s. (default: unlimited)") f.UintVar(&globalOptions.PackSize, "pack-size", 0, "set target pack `size` in MiB, created pack files may be larger (default: $RESTIC_PACK_SIZE)") diff --git a/doc/047_tuning_backup_parameters.rst b/doc/047_tuning_backup_parameters.rst index 5773ac161..d8fb2c9b6 100644 --- a/doc/047_tuning_backup_parameters.rst +++ b/doc/047_tuning_backup_parameters.rst @@ -63,12 +63,15 @@ variable ``RESTIC_COMPRESSION``. Data Verification ================= -To prevent the upload of corrupted data to the repository, restic verifies that files can -be decoded and contain the correct data beforehand. This increases the CPU usage during -backups. If necessary, you can disable this verification using the option ``--no-extra-verify``. -However, in this case you should verify the repository integrity more actively using -``restic check --read-data``. Otherwise, data corruption due to hardware issues or software -bugs might go unnoticed. +To prevent the upload of corrupted data to the repository, which can happen due +to hardware issues or software bugs, restic verifies that generated files can +be decoded and contain the correct data beforehand. This increases the CPU usage +during backups. If necessary, you can disable this verification using the +``--no-extra-verify`` option of the ``backup`` command. However, in this case +you should verify the repository integrity more actively using +``restic check --read-data`` (or the similar ``--read-data-subset`` option). +Otherwise, data corruption due to hardware issues or software bugs might go +unnoticed. File Read Concurrency diff --git a/internal/pack/pack.go b/internal/pack/pack.go index 34e87f1f9..f9e7896e0 100644 --- a/internal/pack/pack.go +++ b/internal/pack/pack.go @@ -87,7 +87,8 @@ func (p *Packer) Finalize() error { encryptedHeader = binary.LittleEndian.AppendUint32(encryptedHeader, uint32(len(encryptedHeader))) if err := verifyHeader(p.k, encryptedHeader, p.blobs); err != nil { - return fmt.Errorf("detected data corruption while writing pack-file header: %w\nCorrupted data is either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", err) + //nolint:revive // ignore linter warnings about error message spelling + return fmt.Errorf("Detected data corruption while writing pack-file header: %w\nCorrupted data is either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", err) } // append the header diff --git a/internal/repository/repository.go b/internal/repository/repository.go index cea43b0d3..40794508f 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -425,7 +425,8 @@ func (r *Repository) saveAndEncrypt(ctx context.Context, t restic.BlobType, data ciphertext = r.key.Seal(ciphertext, nonce, data, nil) if err := r.verifyCiphertext(ciphertext, uncompressedLength, id); err != nil { - return 0, fmt.Errorf("detected data corruption while saving blob %v: %w\nCorrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", id, err) + //nolint:revive // ignore linter warnings about error message spelling + return 0, fmt.Errorf("Detected data corruption while saving blob %v: %w\nCorrupted blobs are either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", id, err) } // find suitable packer and add blob @@ -521,7 +522,8 @@ func (r *Repository) SaveUnpacked(ctx context.Context, t restic.FileType, buf [] ciphertext = r.key.Seal(ciphertext, nonce, p, nil) if err := r.verifyUnpacked(ciphertext, t, buf); err != nil { - return restic.ID{}, fmt.Errorf("detected data corruption while saving file of type %v: %w\nCorrupted data is either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting", t, err) + //nolint:revive // ignore linter warnings about error message spelling + return restic.ID{}, fmt.Errorf("Detected data corruption while saving file of type %v: %w\nCorrupted data is either caused by hardware issues or software bugs. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting.", t, err) } if t == restic.ConfigFile { From 6cd2804bfff06957e835e76d7a67c444d202dcbf Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 4 Feb 2024 19:50:34 +0100 Subject: [PATCH 118/158] Prepare changelog for 0.16.4 --- changelog/{unreleased => 0.16.4_2024-02-04}/issue-4529 | 0 changelog/{unreleased => 0.16.4_2024-02-04}/issue-4677 | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename changelog/{unreleased => 0.16.4_2024-02-04}/issue-4529 (100%) rename changelog/{unreleased => 0.16.4_2024-02-04}/issue-4677 (100%) diff --git a/changelog/unreleased/issue-4529 b/changelog/0.16.4_2024-02-04/issue-4529 similarity index 100% rename from changelog/unreleased/issue-4529 rename to changelog/0.16.4_2024-02-04/issue-4529 diff --git a/changelog/unreleased/issue-4677 b/changelog/0.16.4_2024-02-04/issue-4677 similarity index 100% rename from changelog/unreleased/issue-4677 rename to changelog/0.16.4_2024-02-04/issue-4677 From b0ead75de5d0f50b87bfe5146d5544dd3d92c28f Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 4 Feb 2024 19:50:34 +0100 Subject: [PATCH 119/158] Generate CHANGELOG.md for 0.16.4 --- CHANGELOG.md | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba6fa7a11..b8969a443 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # Table of Contents +* [Changelog for 0.16.4](#changelog-for-restic-0164-2024-02-04) * [Changelog for 0.16.3](#changelog-for-restic-0163-2024-01-14) * [Changelog for 0.16.2](#changelog-for-restic-0162-2023-10-29) * [Changelog for 0.16.1](#changelog-for-restic-0161-2023-10-24) @@ -32,6 +33,57 @@ * [Changelog for 0.6.0](#changelog-for-restic-060-2017-05-29) +# Changelog for restic 0.16.4 (2024-02-04) +The following sections list the changes in restic 0.16.4 relevant to +restic users. The changes are ordered by importance. + +## Summary + + * Fix #4677: Downgrade zstd library to fix rare data corruption at max. compression + * Enh #4529: Add extra verification of data integrity before upload + +## Details + + * Bugfix #4677: Downgrade zstd library to fix rare data corruption at max. compression + + In restic 0.16.3, backups where the compression level was set to `max` (using + `--compression max`) could in rare and very specific circumstances result in + data corruption due to a bug in the library used for compressing data. Restic + 0.16.1 and 0.16.2 were not affected. + + Restic now uses the previous version of the library used to compress data, the + same version used by restic 0.16.2. Please note that the `auto` compression + level (which restic uses by default) was never affected, and even if you used + `max` compression, chances of being affected by this issue are small. + + To check a repository for any corruption, run `restic check --read-data`. This + will download and verify the whole repository and can be used at any time to + completely verify the integrity of a repository. If the `check` command detects + anomalies, follow the suggested steps. + + https://github.com/restic/restic/issues/4677 + https://github.com/restic/restic/pull/4679 + + * Enhancement #4529: Add extra verification of data integrity before upload + + Hardware issues, or a bug in restic or its dependencies, could previously cause + corruption in the files restic created and stored in the repository. Detecting + such corruption previously required explicitly running the `check --read-data` + or `check --read-data-subset` commands. + + To further ensure data integrity, even in the case of hardware issues or + software bugs, restic now performs additional verification of the files about to + be uploaded to the repository. + + These extra checks will increase CPU usage during backups. They can therefore, + if absolutely necessary, be disabled using the `--no-extra-verify` global + option. Please note that this should be combined with more active checking using + the previously mentioned check commands. + + https://github.com/restic/restic/issues/4529 + https://github.com/restic/restic/pull/4681 + + # Changelog for restic 0.16.3 (2024-01-14) The following sections list the changes in restic 0.16.3 relevant to restic users. The changes are ordered by importance. From 811be5984d5d42acf58714d13aeb93cebf4fe2e0 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 4 Feb 2024 19:50:51 +0100 Subject: [PATCH 120/158] Update manpages and auto-completion --- doc/bash-completion.sh | 33 +++++++++++++++++++++++++++++++ doc/man/restic-backup.1 | 4 ++++ doc/man/restic-cache.1 | 4 ++++ doc/man/restic-cat.1 | 4 ++++ doc/man/restic-check.1 | 4 ++++ doc/man/restic-copy.1 | 4 ++++ doc/man/restic-diff.1 | 4 ++++ doc/man/restic-dump.1 | 4 ++++ doc/man/restic-find.1 | 4 ++++ doc/man/restic-forget.1 | 4 ++++ doc/man/restic-generate.1 | 4 ++++ doc/man/restic-init.1 | 4 ++++ doc/man/restic-key.1 | 4 ++++ doc/man/restic-list.1 | 4 ++++ doc/man/restic-ls.1 | 4 ++++ doc/man/restic-migrate.1 | 4 ++++ doc/man/restic-mount.1 | 4 ++++ doc/man/restic-prune.1 | 4 ++++ doc/man/restic-recover.1 | 4 ++++ doc/man/restic-repair-index.1 | 4 ++++ doc/man/restic-repair-packs.1 | 4 ++++ doc/man/restic-repair-snapshots.1 | 4 ++++ doc/man/restic-repair.1 | 4 ++++ doc/man/restic-restore.1 | 4 ++++ doc/man/restic-rewrite.1 | 4 ++++ doc/man/restic-self-update.1 | 4 ++++ doc/man/restic-snapshots.1 | 4 ++++ doc/man/restic-stats.1 | 4 ++++ doc/man/restic-tag.1 | 4 ++++ doc/man/restic-unlock.1 | 4 ++++ doc/man/restic-version.1 | 4 ++++ doc/man/restic.1 | 4 ++++ 32 files changed, 157 insertions(+) diff --git a/doc/bash-completion.sh b/doc/bash-completion.sh index e691af363..cae37a6ca 100644 --- a/doc/bash-completion.sh +++ b/doc/bash-completion.sh @@ -488,6 +488,7 @@ _restic_backup() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -560,6 +561,7 @@ _restic_cache() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -624,6 +626,7 @@ _restic_cat() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -696,6 +699,7 @@ _restic_check() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -794,6 +798,7 @@ _restic_copy() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -860,6 +865,7 @@ _restic_diff() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -944,6 +950,7 @@ _restic_dump() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1058,6 +1065,7 @@ _restic_find() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1228,6 +1236,7 @@ _restic_forget() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1312,6 +1321,7 @@ _restic_generate() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1372,6 +1382,7 @@ _restic_help() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1463,6 +1474,7 @@ _restic_init() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1539,6 +1551,7 @@ _restic_key() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1603,6 +1616,7 @@ _restic_list() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1689,6 +1703,7 @@ _restic_ls() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1757,6 +1772,7 @@ _restic_migrate() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1849,6 +1865,7 @@ _restic_mount() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1935,6 +1952,7 @@ _restic_prune() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -1999,6 +2017,7 @@ _restic_recover() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2059,6 +2078,7 @@ _restic_repair_help() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2126,6 +2146,7 @@ _restic_repair_index() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2190,6 +2211,7 @@ _restic_repair_packs() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2274,6 +2296,7 @@ _restic_repair_snapshots() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2342,6 +2365,7 @@ _restic_repair() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2450,6 +2474,7 @@ _restic_restore() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2552,6 +2577,7 @@ _restic_rewrite() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2620,6 +2646,7 @@ _restic_self-update() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2712,6 +2739,7 @@ _restic_snapshots() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2794,6 +2822,7 @@ _restic_stats() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2884,6 +2913,7 @@ _restic_tag() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -2950,6 +2980,7 @@ _restic_unlock() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -3014,6 +3045,7 @@ _restic_version() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") @@ -3106,6 +3138,7 @@ _restic_root_command() flags+=("--limit-upload=") two_word_flags+=("--limit-upload") flags+=("--no-cache") + flags+=("--no-extra-verify") flags+=("--no-lock") flags+=("--option=") two_word_flags+=("--option") diff --git a/doc/man/restic-backup.1 b/doc/man/restic-backup.1 index c3bccdfa5..730685271 100644 --- a/doc/man/restic-backup.1 +++ b/doc/man/restic-backup.1 @@ -171,6 +171,10 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-cache.1 b/doc/man/restic-cache.1 index 3ae27ea57..c170c1624 100644 --- a/doc/man/restic-cache.1 +++ b/doc/man/restic-cache.1 @@ -80,6 +80,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-cat.1 b/doc/man/restic-cat.1 index c1df138aa..b42a58e14 100644 --- a/doc/man/restic-cat.1 +++ b/doc/man/restic-cat.1 @@ -68,6 +68,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-check.1 b/doc/man/restic-check.1 index 17eb972bc..9c1dc77e5 100644 --- a/doc/man/restic-check.1 +++ b/doc/man/restic-check.1 @@ -85,6 +85,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-copy.1 b/doc/man/restic-copy.1 index be8f21e25..bd9795f44 100644 --- a/doc/man/restic-copy.1 +++ b/doc/man/restic-copy.1 @@ -109,6 +109,10 @@ new destination repository using the "init" command. \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-diff.1 b/doc/man/restic-diff.1 index a01a2562b..28f3a4838 100644 --- a/doc/man/restic-diff.1 +++ b/doc/man/restic-diff.1 @@ -93,6 +93,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-dump.1 b/doc/man/restic-dump.1 index 6fa1f8200..7fa3f777d 100644 --- a/doc/man/restic-dump.1 +++ b/doc/man/restic-dump.1 @@ -96,6 +96,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-find.1 b/doc/man/restic-find.1 index 72bc3a0b6..c3297c43f 100644 --- a/doc/man/restic-find.1 +++ b/doc/man/restic-find.1 @@ -117,6 +117,10 @@ It can also be used to search for restic blobs or trees for troubleshooting. \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-forget.1 b/doc/man/restic-forget.1 index 757022a21..d0c4cfc74 100644 --- a/doc/man/restic-forget.1 +++ b/doc/man/restic-forget.1 @@ -179,6 +179,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-generate.1 b/doc/man/restic-generate.1 index aef3a5e55..84f659ef2 100644 --- a/doc/man/restic-generate.1 +++ b/doc/man/restic-generate.1 @@ -89,6 +89,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-init.1 b/doc/man/restic-init.1 index 27d7f5874..5f19c8f8c 100644 --- a/doc/man/restic-init.1 +++ b/doc/man/restic-init.1 @@ -96,6 +96,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-key.1 b/doc/man/restic-key.1 index 855ef5443..8d1813188 100644 --- a/doc/man/restic-key.1 +++ b/doc/man/restic-key.1 @@ -80,6 +80,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-list.1 b/doc/man/restic-list.1 index 95eeac5f7..e399038a2 100644 --- a/doc/man/restic-list.1 +++ b/doc/man/restic-list.1 @@ -68,6 +68,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-ls.1 b/doc/man/restic-ls.1 index 0cd0f5a88..10b0657a3 100644 --- a/doc/man/restic-ls.1 +++ b/doc/man/restic-ls.1 @@ -107,6 +107,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-migrate.1 b/doc/man/restic-migrate.1 index eca0ef8e1..7e48f726c 100644 --- a/doc/man/restic-migrate.1 +++ b/doc/man/restic-migrate.1 @@ -74,6 +74,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-mount.1 b/doc/man/restic-mount.1 index 33c016ffa..aab607fcf 100644 --- a/doc/man/restic-mount.1 +++ b/doc/man/restic-mount.1 @@ -144,6 +144,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-prune.1 b/doc/man/restic-prune.1 index e4a32cac3..c54d5d7ff 100644 --- a/doc/man/restic-prune.1 +++ b/doc/man/restic-prune.1 @@ -97,6 +97,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-recover.1 b/doc/man/restic-recover.1 index 26d2fc7bd..010fbafd7 100644 --- a/doc/man/restic-recover.1 +++ b/doc/man/restic-recover.1 @@ -70,6 +70,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-repair-index.1 b/doc/man/restic-repair-index.1 index 35e2845b8..f06be64c0 100644 --- a/doc/man/restic-repair-index.1 +++ b/doc/man/restic-repair-index.1 @@ -73,6 +73,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-repair-packs.1 b/doc/man/restic-repair-packs.1 index b21211925..f3671fe18 100644 --- a/doc/man/restic-repair-packs.1 +++ b/doc/man/restic-repair-packs.1 @@ -72,6 +72,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-repair-snapshots.1 b/doc/man/restic-repair-snapshots.1 index f59067f05..9369f25f2 100644 --- a/doc/man/restic-repair-snapshots.1 +++ b/doc/man/restic-repair-snapshots.1 @@ -107,6 +107,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-repair.1 b/doc/man/restic-repair.1 index dbe783df4..77aecc173 100644 --- a/doc/man/restic-repair.1 +++ b/doc/man/restic-repair.1 @@ -63,6 +63,10 @@ Repair the repository \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-restore.1 b/doc/man/restic-restore.1 index d8c1b72e1..4635b1e43 100644 --- a/doc/man/restic-restore.1 +++ b/doc/man/restic-restore.1 @@ -117,6 +117,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-rewrite.1 b/doc/man/restic-rewrite.1 index 8a06aef40..d63c653e6 100644 --- a/doc/man/restic-rewrite.1 +++ b/doc/man/restic-rewrite.1 @@ -121,6 +121,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-self-update.1 b/doc/man/restic-self-update.1 index 28fd24a92..92ab5add3 100644 --- a/doc/man/restic-self-update.1 +++ b/doc/man/restic-self-update.1 @@ -75,6 +75,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-snapshots.1 b/doc/man/restic-snapshots.1 index cb34d6c8e..6203bbf2b 100644 --- a/doc/man/restic-snapshots.1 +++ b/doc/man/restic-snapshots.1 @@ -92,6 +92,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-stats.1 b/doc/man/restic-stats.1 index cf0374351..9d37163de 100644 --- a/doc/man/restic-stats.1 +++ b/doc/man/restic-stats.1 @@ -114,6 +114,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-tag.1 b/doc/man/restic-tag.1 index 162d50d29..b1468c74d 100644 --- a/doc/man/restic-tag.1 +++ b/doc/man/restic-tag.1 @@ -99,6 +99,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-unlock.1 b/doc/man/restic-unlock.1 index 0274c56e8..0b3b43f2a 100644 --- a/doc/man/restic-unlock.1 +++ b/doc/man/restic-unlock.1 @@ -72,6 +72,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic-version.1 b/doc/man/restic-version.1 index 774e19453..ccc23038f 100644 --- a/doc/man/restic-version.1 +++ b/doc/man/restic-version.1 @@ -69,6 +69,10 @@ Exit status is 0 if the command was successful, and non-zero if there was any er \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories diff --git a/doc/man/restic.1 b/doc/man/restic.1 index 427ce7c65..333eab76a 100644 --- a/doc/man/restic.1 +++ b/doc/man/restic.1 @@ -65,6 +65,10 @@ The full documentation can be found at https://restic.readthedocs.io/ . \fB--no-cache\fP[=false] do not use a local cache +.PP +\fB--no-extra-verify\fP[=false] + skip additional verification of data before upload (see documentation) + .PP \fB--no-lock\fP[=false] do not lock the repository, this allows some operations on read-only repositories From 3786536dc18ef27aedcfa8e4c6953b48353eee79 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 4 Feb 2024 19:50:52 +0100 Subject: [PATCH 121/158] Add version for 0.16.4 --- VERSION | 2 +- cmd/restic/global.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 7eb3095a3..5f2491c5a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.16.3 +0.16.4 diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 528c6e129..e979dcc2b 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -43,7 +43,7 @@ import ( "golang.org/x/term" ) -var version = "0.16.3-dev (compiled manually)" +var version = "0.16.4" // TimeFormat is the format used for all timestamps printed by restic. const TimeFormat = "2006-01-02 15:04:05" From 0f9fa44de5a27efd22a17c9e49669a29d5ecf33a Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sun, 4 Feb 2024 19:50:56 +0100 Subject: [PATCH 122/158] Set development version for 0.16.4 --- cmd/restic/global.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/global.go b/cmd/restic/global.go index e979dcc2b..c179e8d33 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -43,7 +43,7 @@ import ( "golang.org/x/term" ) -var version = "0.16.4" +var version = "0.16.4-dev (compiled manually)" // TimeFormat is the format used for all timestamps printed by restic. const TimeFormat = "2006-01-02 15:04:05" From 5771c4ecfbd1d122d61684b678d51eea0af957ab Mon Sep 17 00:00:00 2001 From: Srigovind Nayak Date: Sun, 4 Feb 2024 16:33:00 +0530 Subject: [PATCH 123/158] docs: update environment variables for `az login` to azure backend --- doc/030_preparing_a_new_repo.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index 04fa45468..8661f5904 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -549,6 +549,14 @@ For authentication export one of the following variables: # For SAS $ export AZURE_ACCOUNT_SAS= +For authentication using ``az login`` set the resource group name and ensure the user has +the minimum permissions of the role assignment ``Storage Blob Data Contributor`` on Azure RBAC. + +.. code-block:: console + + $ export AZURE_RESOURCE_GROUP= + $ az login + Alternatively, if run on Azure, restic will automatically uses service accounts configured via the standard environment variables or Workload / Managed Identities. From bca099ac7f469f501a3d611dab045e465993d59d Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 5 Feb 2024 19:52:56 +0100 Subject: [PATCH 124/158] Upgrade zstd library to latest version The data corruption bug is fixed, thus remove the override. --- go.mod | 4 +--- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index fca1fddf2..afcbc427b 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/go-ole/go-ole v1.3.0 github.com/google/go-cmp v0.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 - github.com/klauspost/compress v1.17.4 + github.com/klauspost/compress v1.17.6 github.com/minio/minio-go/v7 v7.0.66 github.com/minio/sha256-simd v1.0.1 github.com/ncw/swift/v2 v2.0.2 @@ -36,8 +36,6 @@ require ( google.golang.org/api v0.157.0 ) -replace github.com/klauspost/compress => github.com/klauspost/compress v1.17.2 - require ( cloud.google.com/go v0.112.0 // indirect cloud.google.com/go/compute v1.23.3 // indirect diff --git a/go.sum b/go.sum index f8989cdd0..fb70ac9d5 100644 --- a/go.sum +++ b/go.sum @@ -117,8 +117,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= -github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI= +github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= From 175c14b5c95e87c98b8a9ef94a379a501bd3f6b2 Mon Sep 17 00:00:00 2001 From: Srigovind Nayak Date: Sun, 4 Feb 2024 14:15:14 +0530 Subject: [PATCH 125/158] dump: add --target option --- changelog/unreleased/issue-4678 | 7 +++++++ cmd/restic/cmd_dump.go | 29 +++++++++++++++++++++++------ doc/050_restore.rst | 7 +++++++ 3 files changed, 37 insertions(+), 6 deletions(-) create mode 100644 changelog/unreleased/issue-4678 diff --git a/changelog/unreleased/issue-4678 b/changelog/unreleased/issue-4678 new file mode 100644 index 000000000..da669fa82 --- /dev/null +++ b/changelog/unreleased/issue-4678 @@ -0,0 +1,7 @@ +Enhancement: Add --target flag to the dump command + +Restic `dump` always printed to the standard output. It now permits to select a +`--target` file to print the output. + +https://github.com/restic/restic/issues/4678 +https://github.com/restic/restic/pull/4682 diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index e72b51fa2..cf2f305c5 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -46,6 +46,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er type DumpOptions struct { restic.SnapshotFilter Archive string + Target string } var dumpOptions DumpOptions @@ -56,6 +57,7 @@ func init() { flags := cmdDump.Flags() initSingleSnapshotFilter(flags, &dumpOptions.SnapshotFilter) flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"") + flags.StringVarP(&dumpOptions.Target, "target", "t", "", "set the target path to dump the archive file") } func splitPath(p string) []string { @@ -67,11 +69,11 @@ func splitPath(p string) []string { return append(s, f) } -func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper) error { +func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, checkStdoutArchiveFunc func() error) error { // If we print / we need to assume that there are multiple nodes at that // level in the tree. if pathComponents[0] == "" { - if err := checkStdoutArchive(); err != nil { + if err := checkStdoutArchiveFunc(); err != nil { return err } return d.DumpTree(ctx, tree, "/") @@ -91,9 +93,9 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoade if err != nil { return errors.Wrapf(err, "cannot load subtree for %q", item) } - return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d) + return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d, checkStdoutArchiveFunc) case dump.IsDir(node): - if err := checkStdoutArchive(); err != nil { + if err := checkStdoutArchiveFunc(); err != nil { return err } subtree, err := restic.LoadTree(ctx, repo, *node.Subtree) @@ -168,8 +170,23 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] return errors.Fatalf("loading tree for snapshot %q failed: %v", snapshotIDString, err) } - d := dump.New(opts.Archive, repo, os.Stdout) - err = printFromTree(ctx, tree, repo, "/", splittedPath, d) + var outputFileWriter = os.Stdout + checkStdoutArchiveFunc := checkStdoutArchive + if opts.Target != "" { + file, err := os.OpenFile(opts.Target, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o666) + if err != nil { + return fmt.Errorf("cannot dump to file: %w", err) + } + defer func() { + _ = file.Close() + }() + + outputFileWriter = file + checkStdoutArchiveFunc = func() error { return nil } + } + + d := dump.New(opts.Archive, repo, outputFileWriter) + err = printFromTree(ctx, tree, repo, "/", splittedPath, d, checkStdoutArchiveFunc) if err != nil { return errors.Fatalf("cannot dump file: %v", err) } diff --git a/doc/050_restore.rst b/doc/050_restore.rst index 56f6458ed..7ff7f2706 100644 --- a/doc/050_restore.rst +++ b/doc/050_restore.rst @@ -174,3 +174,10 @@ To include the folder content at the root of the archive, you can use the `` restore.tar + +It is also possible to ``dump`` the contents of a selected snapshot and folder +structure to a file using the ``--target`` flag. The ``dump`` will fail if the +file exists. + +.. code-block:: console + $ restic -r /srv/restic-repo dump latest / --target /home/linux.user/output.tar -a tar \ No newline at end of file From f5ffa40652da68ec1b2010f592827482d3b807c4 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 5 Feb 2024 20:09:58 +0100 Subject: [PATCH 126/158] dump: minor cleanups --- changelog/unreleased/issue-4678 | 2 +- cmd/restic/cmd_dump.go | 19 ++++++++++--------- doc/050_restore.rst | 4 ++-- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/changelog/unreleased/issue-4678 b/changelog/unreleased/issue-4678 index da669fa82..f98711ea4 100644 --- a/changelog/unreleased/issue-4678 +++ b/changelog/unreleased/issue-4678 @@ -1,7 +1,7 @@ Enhancement: Add --target flag to the dump command Restic `dump` always printed to the standard output. It now permits to select a -`--target` file to print the output. +`--target` file to write the output to. https://github.com/restic/restic/issues/4678 https://github.com/restic/restic/pull/4682 diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index cf2f305c5..1628d6a95 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -57,7 +57,7 @@ func init() { flags := cmdDump.Flags() initSingleSnapshotFilter(flags, &dumpOptions.SnapshotFilter) flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"") - flags.StringVarP(&dumpOptions.Target, "target", "t", "", "set the target path to dump the archive file") + flags.StringVarP(&dumpOptions.Target, "target", "t", "", "write the output to target `path`") } func splitPath(p string) []string { @@ -69,11 +69,11 @@ func splitPath(p string) []string { return append(s, f) } -func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, checkStdoutArchiveFunc func() error) error { +func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, canWriteArchiveFunc func() error) error { // If we print / we need to assume that there are multiple nodes at that // level in the tree. if pathComponents[0] == "" { - if err := checkStdoutArchiveFunc(); err != nil { + if err := canWriteArchiveFunc(); err != nil { return err } return d.DumpTree(ctx, tree, "/") @@ -93,9 +93,9 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoade if err != nil { return errors.Wrapf(err, "cannot load subtree for %q", item) } - return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d, checkStdoutArchiveFunc) + return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d, canWriteArchiveFunc) case dump.IsDir(node): - if err := checkStdoutArchiveFunc(); err != nil { + if err := canWriteArchiveFunc(); err != nil { return err } subtree, err := restic.LoadTree(ctx, repo, *node.Subtree) @@ -170,8 +170,9 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] return errors.Fatalf("loading tree for snapshot %q failed: %v", snapshotIDString, err) } - var outputFileWriter = os.Stdout - checkStdoutArchiveFunc := checkStdoutArchive + outputFileWriter := os.Stdout + canWriteArchiveFunc := checkStdoutArchive + if opts.Target != "" { file, err := os.OpenFile(opts.Target, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o666) if err != nil { @@ -182,11 +183,11 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] }() outputFileWriter = file - checkStdoutArchiveFunc = func() error { return nil } + canWriteArchiveFunc = func() error { return nil } } d := dump.New(opts.Archive, repo, outputFileWriter) - err = printFromTree(ctx, tree, repo, "/", splittedPath, d, checkStdoutArchiveFunc) + err = printFromTree(ctx, tree, repo, "/", splittedPath, d, canWriteArchiveFunc) if err != nil { return errors.Fatalf("cannot dump file: %v", err) } diff --git a/doc/050_restore.rst b/doc/050_restore.rst index 7ff7f2706..3cd05500b 100644 --- a/doc/050_restore.rst +++ b/doc/050_restore.rst @@ -176,8 +176,8 @@ To include the folder content at the root of the archive, you can use the `` restore.tar It is also possible to ``dump`` the contents of a selected snapshot and folder -structure to a file using the ``--target`` flag. The ``dump`` will fail if the -file exists. +structure to a file using the ``--target`` flag. The ``dump`` command will fail +if the already file exists. .. code-block:: console $ restic -r /srv/restic-repo dump latest / --target /home/linux.user/output.tar -a tar \ No newline at end of file From 795d33b3ee5772c8cfc8e85621d131dc9cc3169b Mon Sep 17 00:00:00 2001 From: Srigovind Nayak Date: Sun, 4 Feb 2024 20:10:23 +0530 Subject: [PATCH 127/158] key: move add, list, remove, passwd to sub-commands docs: improve the sub-command docs changelog: add the unreleased changelog for the key command updates key: update integration tests --- changelog/unreleased/issue-4676 | 8 + cmd/restic/cmd_key.go | 254 +------------------------ cmd/restic/cmd_key_add.go | 128 +++++++++++++ cmd/restic/cmd_key_integration_test.go | 55 +++++- cmd/restic/cmd_key_list.go | 112 +++++++++++ cmd/restic/cmd_key_passwd.go | 89 +++++++++ cmd/restic/cmd_key_remove.go | 73 +++++++ 7 files changed, 459 insertions(+), 260 deletions(-) create mode 100644 changelog/unreleased/issue-4676 create mode 100644 cmd/restic/cmd_key_add.go create mode 100644 cmd/restic/cmd_key_list.go create mode 100644 cmd/restic/cmd_key_passwd.go create mode 100644 cmd/restic/cmd_key_remove.go diff --git a/changelog/unreleased/issue-4676 b/changelog/unreleased/issue-4676 new file mode 100644 index 000000000..e95118e72 --- /dev/null +++ b/changelog/unreleased/issue-4676 @@ -0,0 +1,8 @@ +Enhancement: Move key add, list, remove and passwd as separate sub-commands + +Restic now provides usage documentation for the `key` command. Each sub-command; +`add`, `list`, `remove` and `passwd` now have their own sub-command documentation +which can be invoked using `restic key --help`. + +https://github.com/restic/restic/issues/4676 +https://github.com/restic/restic/pull/4685 diff --git a/cmd/restic/cmd_key.go b/cmd/restic/cmd_key.go index 14609e6e9..c687eca53 100644 --- a/cmd/restic/cmd_key.go +++ b/cmd/restic/cmd_key.go @@ -1,264 +1,18 @@ package main import ( - "context" - "encoding/json" - "os" - "strings" - "sync" - - "github.com/restic/restic/internal/errors" - "github.com/restic/restic/internal/repository" - "github.com/restic/restic/internal/restic" - "github.com/restic/restic/internal/ui/table" - "github.com/spf13/cobra" ) var cmdKey = &cobra.Command{ - Use: "key [flags] [list|add|remove|passwd] [ID]", + Use: "key", Short: "Manage keys (passwords)", Long: ` -The "key" command manages keys (passwords) for accessing the repository. - -EXIT STATUS -=========== - -Exit status is 0 if the command was successful, and non-zero if there was any error. -`, - DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { - return runKey(cmd.Context(), globalOptions, keyOpts, args) - }, +The "key" command allows you to set multiple access keys or passwords +per repository. + `, } -type KeyOptions struct { - NewPasswordFile string - Username string - Hostname string -} - -var keyOpts KeyOptions - func init() { cmdRoot.AddCommand(cmdKey) - - flags := cmdKey.Flags() - flags.StringVarP(&keyOpts.NewPasswordFile, "new-password-file", "", "", "`file` from which to read the new password") - flags.StringVarP(&keyOpts.Username, "user", "", "", "the username for new keys") - flags.StringVarP(&keyOpts.Hostname, "host", "", "", "the hostname for new keys") -} - -func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions) error { - type keyInfo struct { - Current bool `json:"current"` - ID string `json:"id"` - UserName string `json:"userName"` - HostName string `json:"hostName"` - Created string `json:"created"` - } - - var m sync.Mutex - var keys []keyInfo - - err := restic.ParallelList(ctx, s, restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, size int64) error { - k, err := repository.LoadKey(ctx, s, id) - if err != nil { - Warnf("LoadKey() failed: %v\n", err) - return nil - } - - key := keyInfo{ - Current: id == s.KeyID(), - ID: id.Str(), - UserName: k.Username, - HostName: k.Hostname, - Created: k.Created.Local().Format(TimeFormat), - } - - m.Lock() - defer m.Unlock() - keys = append(keys, key) - return nil - }) - - if err != nil { - return err - } - - if gopts.JSON { - return json.NewEncoder(globalOptions.stdout).Encode(keys) - } - - tab := table.New() - tab.AddColumn(" ID", "{{if .Current}}*{{else}} {{end}}{{ .ID }}") - tab.AddColumn("User", "{{ .UserName }}") - tab.AddColumn("Host", "{{ .HostName }}") - tab.AddColumn("Created", "{{ .Created }}") - - for _, key := range keys { - tab.AddRow(key) - } - - return tab.Write(globalOptions.stdout) -} - -// testKeyNewPassword is used to set a new password during integration testing. -var testKeyNewPassword string - -func getNewPassword(gopts GlobalOptions, newPasswordFile string) (string, error) { - if testKeyNewPassword != "" { - return testKeyNewPassword, nil - } - - if newPasswordFile != "" { - return loadPasswordFromFile(newPasswordFile) - } - - // Since we already have an open repository, temporary remove the password - // to prompt the user for the passwd. - newopts := gopts - newopts.password = "" - - return ReadPasswordTwice(newopts, - "enter new password: ", - "enter password again: ") -} - -func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyOptions) error { - pw, err := getNewPassword(gopts, opts.NewPasswordFile) - if err != nil { - return err - } - - id, err := repository.AddKey(ctx, repo, pw, opts.Username, opts.Hostname, repo.Key()) - if err != nil { - return errors.Fatalf("creating new key failed: %v\n", err) - } - - err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw) - if err != nil { - return err - } - - Verbosef("saved new key with ID %s\n", id.ID()) - - return nil -} - -func deleteKey(ctx context.Context, repo *repository.Repository, id restic.ID) error { - if id == repo.KeyID() { - return errors.Fatal("refusing to remove key currently used to access repository") - } - - err := repository.RemoveKey(ctx, repo, id) - if err != nil { - return err - } - - Verbosef("removed key %v\n", id) - return nil -} - -func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, newPasswordFile string) error { - pw, err := getNewPassword(gopts, newPasswordFile) - if err != nil { - return err - } - - id, err := repository.AddKey(ctx, repo, pw, "", "", repo.Key()) - if err != nil { - return errors.Fatalf("creating new key failed: %v\n", err) - } - oldID := repo.KeyID() - - err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw) - if err != nil { - return err - } - - err = repository.RemoveKey(ctx, repo, oldID) - if err != nil { - return err - } - - Verbosef("saved new key as %s\n", id) - - return nil -} - -func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error { - // Verify new key to make sure it really works. A broken key can render the - // whole repository inaccessible - err := repo.SearchKey(ctx, pw, 0, key.ID().String()) - if err != nil { - // the key is invalid, try to remove it - _ = repository.RemoveKey(ctx, repo, key.ID()) - return errors.Fatalf("failed to access repository with new key: %v", err) - } - return nil -} - -func runKey(ctx context.Context, gopts GlobalOptions, opts KeyOptions, args []string) error { - if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) { - return errors.Fatal("wrong number of arguments") - } - - repo, err := OpenRepository(ctx, gopts) - if err != nil { - return err - } - - switch args[0] { - case "list": - if !gopts.NoLock { - var lock *restic.Lock - lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - } - - return listKeys(ctx, repo, gopts) - case "add": - lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - - return addKey(ctx, repo, gopts, opts) - case "remove": - lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - - id, err := restic.Find(ctx, repo, restic.KeyFile, args[1]) - if err != nil { - return err - } - - return deleteKey(ctx, repo, id) - case "passwd": - lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) - defer unlockRepo(lock) - if err != nil { - return err - } - - return changePassword(ctx, repo, gopts, opts.NewPasswordFile) - default: - return errors.Fatal("invalid operation") - } -} - -func loadPasswordFromFile(pwdFile string) (string, error) { - s, err := os.ReadFile(pwdFile) - if os.IsNotExist(err) { - return "", errors.Fatalf("%s does not exist", pwdFile) - } - return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile") } diff --git a/cmd/restic/cmd_key_add.go b/cmd/restic/cmd_key_add.go new file mode 100644 index 000000000..43a38f4eb --- /dev/null +++ b/cmd/restic/cmd_key_add.go @@ -0,0 +1,128 @@ +package main + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" + "github.com/spf13/cobra" +) + +var cmdKeyAdd = &cobra.Command{ + Use: "add", + Short: "Add a new key (password) to the repository; returns the new key ID", + Long: ` +The "add" sub-command creates a new key and validates the key. Returns the new key ID. + +EXIT STATUS +=========== + +Exit status is 0 if the command is successful, and non-zero if there was any error. + `, + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + return runKeyAdd(cmd.Context(), globalOptions, keyAddOpts, args) + }, +} + +type KeyAddOptions struct { + NewPasswordFile string + Username string + Hostname string +} + +var keyAddOpts KeyAddOptions + +func init() { + cmdKey.AddCommand(cmdKeyAdd) + + flags := cmdKeyAdd.Flags() + flags.StringVarP(&keyAddOpts.NewPasswordFile, "new-password-file", "", "", "`file` from which to read the new password") + flags.StringVarP(&keyAddOpts.Username, "user", "", "", "the username for new key") + flags.StringVarP(&keyAddOpts.Hostname, "host", "", "", "the hostname for new key") +} + +func runKeyAdd(ctx context.Context, gopts GlobalOptions, opts KeyAddOptions, args []string) error { + if len(args) > 0 { + return fmt.Errorf("the key add command expects no arguments, only options - please see `restic help key add` for usage and flags") + } + + repo, err := OpenRepository(ctx, gopts) + if err != nil { + return err + } + + lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) + defer unlockRepo(lock) + if err != nil { + return err + } + + return addKey(ctx, repo, gopts, opts) +} + +func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyAddOptions) error { + pw, err := getNewPassword(gopts, opts.NewPasswordFile) + if err != nil { + return err + } + + id, err := repository.AddKey(ctx, repo, pw, opts.Username, opts.Hostname, repo.Key()) + if err != nil { + return errors.Fatalf("creating new key failed: %v\n", err) + } + + err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw) + if err != nil { + return err + } + + Verbosef("saved new key with ID %s\n", id.ID()) + + return nil +} + +// testKeyNewPassword is used to set a new password during integration testing. +var testKeyNewPassword string + +func getNewPassword(gopts GlobalOptions, newPasswordFile string) (string, error) { + if testKeyNewPassword != "" { + return testKeyNewPassword, nil + } + + if newPasswordFile != "" { + return loadPasswordFromFile(newPasswordFile) + } + + // Since we already have an open repository, temporary remove the password + // to prompt the user for the passwd. + newopts := gopts + newopts.password = "" + + return ReadPasswordTwice(newopts, + "enter new password: ", + "enter password again: ") +} + +func loadPasswordFromFile(pwdFile string) (string, error) { + s, err := os.ReadFile(pwdFile) + if os.IsNotExist(err) { + return "", errors.Fatalf("%s does not exist", pwdFile) + } + return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile") +} + +func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error { + // Verify new key to make sure it really works. A broken key can render the + // whole repository inaccessible + err := repo.SearchKey(ctx, pw, 0, key.ID().String()) + if err != nil { + // the key is invalid, try to remove it + _ = repository.RemoveKey(ctx, repo, key.ID()) + return errors.Fatalf("failed to access repository with new key: %v", err) + } + return nil +} diff --git a/cmd/restic/cmd_key_integration_test.go b/cmd/restic/cmd_key_integration_test.go index 34474c3af..72fa914da 100644 --- a/cmd/restic/cmd_key_integration_test.go +++ b/cmd/restic/cmd_key_integration_test.go @@ -13,7 +13,7 @@ import ( func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string { buf, err := withCaptureStdout(func() error { - return runKey(context.TODO(), gopts, KeyOptions{}, []string{"list"}) + return runKeyList(context.TODO(), gopts, []string{}) }) rtest.OK(t, err) @@ -36,7 +36,7 @@ func testRunKeyAddNewKey(t testing.TB, newPassword string, gopts GlobalOptions) testKeyNewPassword = "" }() - rtest.OK(t, runKey(context.TODO(), gopts, KeyOptions{}, []string{"add"})) + rtest.OK(t, runKeyAdd(context.TODO(), gopts, KeyAddOptions{}, []string{})) } func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) { @@ -46,10 +46,10 @@ func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) { }() t.Log("adding key for john@example.com") - rtest.OK(t, runKey(context.TODO(), gopts, KeyOptions{ + rtest.OK(t, runKeyAdd(context.TODO(), gopts, KeyAddOptions{ Username: "john", Hostname: "example.com", - }, []string{"add"})) + }, []string{})) repo, err := OpenRepository(context.TODO(), gopts) rtest.OK(t, err) @@ -66,13 +66,13 @@ func testRunKeyPasswd(t testing.TB, newPassword string, gopts GlobalOptions) { testKeyNewPassword = "" }() - rtest.OK(t, runKey(context.TODO(), gopts, KeyOptions{}, []string{"passwd"})) + rtest.OK(t, runKeyPasswd(context.TODO(), gopts, KeyPasswdOptions{}, []string{})) } func testRunKeyRemove(t testing.TB, gopts GlobalOptions, IDs []string) { t.Logf("remove %d keys: %q\n", len(IDs), IDs) for _, id := range IDs { - rtest.OK(t, runKey(context.TODO(), gopts, KeyOptions{}, []string{"remove", id})) + rtest.OK(t, runKeyRemove(context.TODO(), gopts, []string{id})) } } @@ -102,7 +102,7 @@ func TestKeyAddRemove(t *testing.T) { env.gopts.password = passwordList[len(passwordList)-1] t.Logf("testing access with last password %q\n", env.gopts.password) - rtest.OK(t, runKey(context.TODO(), env.gopts, KeyOptions{}, []string{"list"})) + rtest.OK(t, runKeyList(context.TODO(), env.gopts, []string{})) testRunCheck(t, env.gopts) testRunKeyAddNewKeyUserHost(t, env.gopts) @@ -130,15 +130,50 @@ func TestKeyProblems(t *testing.T) { testKeyNewPassword = "" }() - err := runKey(context.TODO(), env.gopts, KeyOptions{}, []string{"passwd"}) + err := runKeyPasswd(context.TODO(), env.gopts, KeyPasswdOptions{}, []string{}) t.Log(err) rtest.Assert(t, err != nil, "expected passwd change to fail") - err = runKey(context.TODO(), env.gopts, KeyOptions{}, []string{"add"}) + err = runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{}, []string{}) t.Log(err) rtest.Assert(t, err != nil, "expected key adding to fail") t.Logf("testing access with initial password %q\n", env.gopts.password) - rtest.OK(t, runKey(context.TODO(), env.gopts, KeyOptions{}, []string{"list"})) + rtest.OK(t, runKeyList(context.TODO(), env.gopts, []string{})) testRunCheck(t, env.gopts) } + +func TestKeyCommandInvalidArguments(t *testing.T) { + env, cleanup := withTestEnvironment(t) + defer cleanup() + + testRunInit(t, env.gopts) + env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { + return &emptySaveBackend{r}, nil + } + + err := runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{}, []string{"johndoe"}) + t.Log(err) + rtest.Assert(t, err != nil, "expected key add to fail") + + testKeyNewPassword = "johndoe" + defer func() { + testKeyNewPassword = "" + }() + err = runKeyPasswd(context.TODO(), env.gopts, KeyPasswdOptions{}, []string{"johndoe"}) + t.Log(err) + rtest.Assert(t, err != nil, "expected key passwd to fail") + + env.gopts.password = "johndoe" + err = runKeyList(context.TODO(), env.gopts, []string{}) + t.Log(err) + rtest.Assert(t, err != nil, "expected key list to fail") + + err = runKeyRemove(context.TODO(), env.gopts, []string{}) + t.Log(err) + rtest.Assert(t, err != nil, "expected key remove to fail") + + err = runKeyRemove(context.TODO(), env.gopts, []string{"john", "doe"}) + t.Log(err) + rtest.Assert(t, err != nil, "expected key remove to fail") +} diff --git a/cmd/restic/cmd_key_list.go b/cmd/restic/cmd_key_list.go new file mode 100644 index 000000000..517b7c84b --- /dev/null +++ b/cmd/restic/cmd_key_list.go @@ -0,0 +1,112 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "sync" + + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/ui/table" + "github.com/spf13/cobra" +) + +var cmdKeyList = &cobra.Command{ + Use: "list", + Short: "List keys (passwords)", + Long: ` +The "list" sub-command lists all the keys (passwords) associated with the repository. +Returns the key ID, username, hostname, created time and if it's the current key being +used to access the repository. + +EXIT STATUS +=========== + +Exit status is 0 if the command is successful, and non-zero if there was any error. + `, + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + return runKeyList(cmd.Context(), globalOptions, args) + }, +} + +func init() { + cmdKey.AddCommand(cmdKeyList) +} + +func runKeyList(ctx context.Context, gopts GlobalOptions, args []string) error { + if len(args) > 0 { + return fmt.Errorf("the key list command expects no arguments, only options - please see `restic help key list` for usage and flags") + } + + repo, err := OpenRepository(ctx, gopts) + if err != nil { + return err + } + + if !gopts.NoLock { + var lock *restic.Lock + lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON) + defer unlockRepo(lock) + if err != nil { + return err + } + } + + return listKeys(ctx, repo, gopts) +} + +func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions) error { + type keyInfo struct { + Current bool `json:"current"` + ID string `json:"id"` + UserName string `json:"userName"` + HostName string `json:"hostName"` + Created string `json:"created"` + } + + var m sync.Mutex + var keys []keyInfo + + err := restic.ParallelList(ctx, s, restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, size int64) error { + k, err := repository.LoadKey(ctx, s, id) + if err != nil { + Warnf("LoadKey() failed: %v\n", err) + return nil + } + + key := keyInfo{ + Current: id == s.KeyID(), + ID: id.Str(), + UserName: k.Username, + HostName: k.Hostname, + Created: k.Created.Local().Format(TimeFormat), + } + + m.Lock() + defer m.Unlock() + keys = append(keys, key) + return nil + }) + + if err != nil { + return err + } + + if gopts.JSON { + return json.NewEncoder(globalOptions.stdout).Encode(keys) + } + + tab := table.New() + tab.AddColumn(" ID", "{{if .Current}}*{{else}} {{end}}{{ .ID }}") + tab.AddColumn("User", "{{ .UserName }}") + tab.AddColumn("Host", "{{ .HostName }}") + tab.AddColumn("Created", "{{ .Created }}") + + for _, key := range keys { + tab.AddRow(key) + } + + return tab.Write(globalOptions.stdout) +} diff --git a/cmd/restic/cmd_key_passwd.go b/cmd/restic/cmd_key_passwd.go new file mode 100644 index 000000000..cb916274c --- /dev/null +++ b/cmd/restic/cmd_key_passwd.go @@ -0,0 +1,89 @@ +package main + +import ( + "context" + "fmt" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" + "github.com/spf13/cobra" +) + +var cmdKeyPasswd = &cobra.Command{ + Use: "passwd", + Short: "Change key (password); creates a new key ID and removes the old key ID, returns new key ID", + Long: ` +The "passwd" sub-command creates a new key, validates the key and remove the old key ID. +Returns the new key ID. + +EXIT STATUS +=========== + +Exit status is 0 if the command is successful, and non-zero if there was any error. + `, + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + return runKeyPasswd(cmd.Context(), globalOptions, keyPasswdOpts, args) + }, +} + +type KeyPasswdOptions struct { + KeyAddOptions +} + +var keyPasswdOpts KeyPasswdOptions + +func init() { + cmdKey.AddCommand(cmdKeyPasswd) + + flags := cmdKeyPasswd.Flags() + flags.StringVarP(&keyPasswdOpts.NewPasswordFile, "new-password-file", "", "", "`file` from which to read the new password") + flags.StringVarP(&keyPasswdOpts.Username, "user", "", "", "the username for new key") + flags.StringVarP(&keyPasswdOpts.Hostname, "host", "", "", "the hostname for new key") +} + +func runKeyPasswd(ctx context.Context, gopts GlobalOptions, opts KeyPasswdOptions, args []string) error { + if len(args) > 0 { + return fmt.Errorf("the key passwd command expects no arguments, only options - please see `restic help key passwd` for usage and flags") + } + + repo, err := OpenRepository(ctx, gopts) + if err != nil { + return err + } + + lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) + defer unlockRepo(lock) + if err != nil { + return err + } + + return changePassword(ctx, repo, gopts, opts) +} + +func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyPasswdOptions) error { + pw, err := getNewPassword(gopts, opts.NewPasswordFile) + if err != nil { + return err + } + + id, err := repository.AddKey(ctx, repo, pw, "", "", repo.Key()) + if err != nil { + return errors.Fatalf("creating new key failed: %v\n", err) + } + oldID := repo.KeyID() + + err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw) + if err != nil { + return err + } + + err = repository.RemoveKey(ctx, repo, oldID) + if err != nil { + return err + } + + Verbosef("saved new key as %s\n", id) + + return nil +} diff --git a/cmd/restic/cmd_key_remove.go b/cmd/restic/cmd_key_remove.go new file mode 100644 index 000000000..c8e303ffc --- /dev/null +++ b/cmd/restic/cmd_key_remove.go @@ -0,0 +1,73 @@ +package main + +import ( + "context" + "fmt" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + "github.com/spf13/cobra" +) + +var cmdKeyRemove = &cobra.Command{ + Use: "remove [ID]", + Short: "Remove key ID (password) from the repository.", + Long: ` +The "remove" sub-command removes the selected key ID. The "remove" command does not allow +removing the current key being used to access the repository. + +EXIT STATUS +=========== + +Exit status is 0 if the command is successful, and non-zero if there was any error. + `, + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + return runKeyRemove(cmd.Context(), globalOptions, args) + }, +} + +func init() { + cmdKey.AddCommand(cmdKeyRemove) +} + +func runKeyRemove(ctx context.Context, gopts GlobalOptions, args []string) error { + if len(args) != 1 { + return fmt.Errorf("key remove expects one argument as the key id") + } + + repo, err := OpenRepository(ctx, gopts) + if err != nil { + return err + } + + lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON) + defer unlockRepo(lock) + if err != nil { + return err + } + + idPrefix := args[0] + + return deleteKey(ctx, repo, idPrefix) +} + +func deleteKey(ctx context.Context, repo *repository.Repository, idPrefix string) error { + id, err := restic.Find(ctx, repo, restic.KeyFile, idPrefix) + if err != nil { + return err + } + + if id == repo.KeyID() { + return errors.Fatal("refusing to remove key currently used to access repository") + } + + err = repository.RemoveKey(ctx, repo, id) + if err != nil { + return err + } + + Verbosef("removed key %v\n", id) + return nil +} From e46b21ab80b97cf4dc8780f11fcf9e94e07076ab Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 8 Feb 2024 20:52:30 +0100 Subject: [PATCH 128/158] key: fix integration test for invalid arguments --- cmd/restic/cmd_key_integration_test.go | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/cmd/restic/cmd_key_integration_test.go b/cmd/restic/cmd_key_integration_test.go index 72fa914da..16cc1bdad 100644 --- a/cmd/restic/cmd_key_integration_test.go +++ b/cmd/restic/cmd_key_integration_test.go @@ -4,6 +4,7 @@ import ( "bufio" "context" "regexp" + "strings" "testing" "github.com/restic/restic/internal/backend" @@ -154,26 +155,21 @@ func TestKeyCommandInvalidArguments(t *testing.T) { err := runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{}, []string{"johndoe"}) t.Log(err) - rtest.Assert(t, err != nil, "expected key add to fail") + rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key add: %v", err) - testKeyNewPassword = "johndoe" - defer func() { - testKeyNewPassword = "" - }() err = runKeyPasswd(context.TODO(), env.gopts, KeyPasswdOptions{}, []string{"johndoe"}) t.Log(err) - rtest.Assert(t, err != nil, "expected key passwd to fail") + rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key passwd: %v", err) - env.gopts.password = "johndoe" - err = runKeyList(context.TODO(), env.gopts, []string{}) + err = runKeyList(context.TODO(), env.gopts, []string{"johndoe"}) t.Log(err) - rtest.Assert(t, err != nil, "expected key list to fail") + rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key list: %v", err) err = runKeyRemove(context.TODO(), env.gopts, []string{}) t.Log(err) - rtest.Assert(t, err != nil, "expected key remove to fail") + rtest.Assert(t, err != nil && strings.Contains(err.Error(), "one argument"), "unexpected error for key remove: %v", err) err = runKeyRemove(context.TODO(), env.gopts, []string{"john", "doe"}) t.Log(err) - rtest.Assert(t, err != nil, "expected key remove to fail") + rtest.Assert(t, err != nil && strings.Contains(err.Error(), "one argument"), "unexpected error for key remove: %v", err) } From 80db02fc3521612c7dc7ce86a9e57f17e20ee9df Mon Sep 17 00:00:00 2001 From: lou Date: Thu, 8 Feb 2024 14:11:57 +0800 Subject: [PATCH 129/158] dump flag --target should be allowed to write existing file Signed-off-by: lou --- cmd/restic/cmd_dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/cmd_dump.go b/cmd/restic/cmd_dump.go index 1628d6a95..9178f2abe 100644 --- a/cmd/restic/cmd_dump.go +++ b/cmd/restic/cmd_dump.go @@ -174,7 +174,7 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args [] canWriteArchiveFunc := checkStdoutArchive if opts.Target != "" { - file, err := os.OpenFile(opts.Target, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o666) + file, err := os.Create(opts.Target) if err != nil { return fmt.Errorf("cannot dump to file: %w", err) } From 15d6fa1f8374449346eb86cb4919bf8629671c84 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 10 Feb 2024 18:38:36 +0100 Subject: [PATCH 130/158] dump: update docs for --target option --- changelog/unreleased/issue-4678 | 1 + doc/050_restore.rst | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog/unreleased/issue-4678 b/changelog/unreleased/issue-4678 index f98711ea4..9f9a213e1 100644 --- a/changelog/unreleased/issue-4678 +++ b/changelog/unreleased/issue-4678 @@ -5,3 +5,4 @@ Restic `dump` always printed to the standard output. It now permits to select a https://github.com/restic/restic/issues/4678 https://github.com/restic/restic/pull/4682 +https://github.com/restic/restic/pull/4692 diff --git a/doc/050_restore.rst b/doc/050_restore.rst index 3cd05500b..916b11c86 100644 --- a/doc/050_restore.rst +++ b/doc/050_restore.rst @@ -176,8 +176,7 @@ To include the folder content at the root of the archive, you can use the `` restore.tar It is also possible to ``dump`` the contents of a selected snapshot and folder -structure to a file using the ``--target`` flag. The ``dump`` command will fail -if the already file exists. +structure to a file using the ``--target`` flag. .. code-block:: console $ restic -r /srv/restic-repo dump latest / --target /home/linux.user/output.tar -a tar \ No newline at end of file From 9720935c56b33fa06287ca4153898ded08484295 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 10 Feb 2024 21:56:01 +0100 Subject: [PATCH 131/158] Update Go version for tests to 1.22 --- .github/workflows/tests.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 45681c6c5..abcb07270 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ permissions: contents: read env: - latest_go: "1.21.x" + latest_go: "1.22.x" GO111MODULE: on jobs: @@ -23,34 +23,34 @@ jobs: # list of jobs to run: include: - job_name: Windows - go: 1.21.x + go: 1.22.x os: windows-latest - job_name: macOS - go: 1.21.x + go: 1.22.x os: macOS-latest test_fuse: false - job_name: Linux - go: 1.21.x + go: 1.22.x os: ubuntu-latest test_cloud_backends: true test_fuse: true check_changelog: true - job_name: Linux (race) - go: 1.21.x + go: 1.22.x os: ubuntu-latest test_fuse: true test_opts: "-race" - job_name: Linux - go: 1.20.x + go: 1.21.x os: ubuntu-latest test_fuse: true - job_name: Linux - go: 1.19.x + go: 1.20.x os: ubuntu-latest test_fuse: true From a8cda0119cdec8755f12c27bd07f312338ec15c1 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 10 Feb 2024 22:08:43 +0100 Subject: [PATCH 132/158] Upgrade golangci-lint --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index abcb07270..cc599491c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -255,7 +255,7 @@ jobs: uses: golangci/golangci-lint-action@v3 with: # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version. - version: v1.55.2 + version: v1.56.1 args: --verbose --timeout 5m # only run golangci-lint for pull requests, otherwise ALL hints get From c0514dd8baa6460ea1d3cf70d58a1a421eedcea8 Mon Sep 17 00:00:00 2001 From: Alexander Neumann Date: Sat, 10 Feb 2024 22:58:10 +0100 Subject: [PATCH 133/158] Fix linter errors (except for tests) --- .golangci.yml | 5 +++++ cmd/restic/cmd_backup.go | 2 +- cmd/restic/cmd_cache.go | 2 +- cmd/restic/cmd_check.go | 8 ++++---- cmd/restic/cmd_diff.go | 2 +- cmd/restic/cmd_generate.go | 2 +- cmd/restic/cmd_key_list.go | 2 +- cmd/restic/cmd_list.go | 4 ++-- cmd/restic/cmd_options.go | 2 +- cmd/restic/cmd_prune.go | 8 ++++---- cmd/restic/cmd_recover.go | 4 ++-- cmd/restic/cmd_repair_index.go | 6 +++--- cmd/restic/cmd_repair_snapshots.go | 2 +- cmd/restic/cmd_restore.go | 4 ++-- cmd/restic/cmd_rewrite.go | 2 +- cmd/restic/cmd_stats.go | 2 +- cmd/restic/cmd_unlock.go | 2 +- cmd/restic/cmd_version.go | 2 +- cmd/restic/main.go | 2 +- internal/archiver/archiver.go | 6 +++--- internal/archiver/scanner.go | 8 ++++---- internal/backend/mem/mem_backend.go | 2 +- internal/backend/rclone/backend.go | 2 +- internal/checker/checker.go | 2 +- internal/fs/fs_track.go | 2 +- internal/fuse/snapshots_dirstruct.go | 2 +- internal/index/index_parallel.go | 2 +- internal/repository/key.go | 2 +- internal/repository/repair_pack.go | 2 +- internal/repository/repository.go | 4 ++-- internal/restic/backend_find.go | 2 +- internal/restic/lock.go | 4 ++-- internal/restic/snapshot.go | 2 +- internal/restorer/restorer.go | 6 +++--- internal/ui/backup/progress.go | 2 +- internal/walker/rewriter.go | 4 ++-- 36 files changed, 61 insertions(+), 56 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index c08331401..7dc6a8e7f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -54,3 +54,8 @@ issues: # staticcheck: there's no easy way to replace these packages - "SA1019: \"golang.org/x/crypto/poly1305\" is deprecated" - "SA1019: \"golang.org/x/crypto/openpgp\" is deprecated" + + exclude-rules: + # revive: ignore unused parameters in tests + - path: (_test\.go|testing\.go|backend/.*/tests\.go) + text: "unused-parameter:" \ No newline at end of file diff --git a/cmd/restic/cmd_backup.go b/cmd/restic/cmd_backup.go index be7a2aa3f..318d17796 100644 --- a/cmd/restic/cmd_backup.go +++ b/cmd/restic/cmd_backup.go @@ -42,7 +42,7 @@ Exit status is 0 if the command was successful. Exit status is 1 if there was a fatal error (no snapshot created). Exit status is 3 if some source data could not be read (incomplete snapshot created). `, - PreRun: func(cmd *cobra.Command, args []string) { + PreRun: func(_ *cobra.Command, _ []string) { if backupOptions.Host == "" { hostname, err := os.Hostname() if err != nil { diff --git a/cmd/restic/cmd_cache.go b/cmd/restic/cmd_cache.go index 4a10d1027..354cec288 100644 --- a/cmd/restic/cmd_cache.go +++ b/cmd/restic/cmd_cache.go @@ -28,7 +28,7 @@ EXIT STATUS Exit status is 0 if the command was successful, and non-zero if there was any error. `, DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, args []string) error { return runCache(cacheOptions, globalOptions, args) }, } diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index f04a4fe71..22f462d75 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -38,7 +38,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er RunE: func(cmd *cobra.Command, args []string) error { return runCheck(cmd.Context(), checkOptions, globalOptions, args) }, - PreRunE: func(cmd *cobra.Command, args []string) error { + PreRunE: func(_ *cobra.Command, _ []string) error { return checkFlags(checkOptions) }, } @@ -345,11 +345,11 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args if len(salvagePacks) > 0 { Warnf("\nThe repository contains pack files with damaged blobs. These blobs must be removed to repair the repository. This can be done using the following commands:\n\n") - var strIds []string + var strIDs []string for _, id := range salvagePacks { - strIds = append(strIds, id.String()) + strIDs = append(strIDs, id.String()) } - Warnf("RESTIC_FEATURES=repair-packs-v1 restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIds, " ")) + Warnf("RESTIC_FEATURES=repair-packs-v1 restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIDs, " ")) Warnf("Corrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n") } } diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index 06f2be2ae..3bd29fa67 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -418,7 +418,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args [] } if gopts.Quiet { - c.printChange = func(change *Change) {} + c.printChange = func(_ *Change) {} } stats := &DiffStatsContainer{ diff --git a/cmd/restic/cmd_generate.go b/cmd/restic/cmd_generate.go index 9d1652e93..ba710e708 100644 --- a/cmd/restic/cmd_generate.go +++ b/cmd/restic/cmd_generate.go @@ -21,7 +21,7 @@ EXIT STATUS Exit status is 0 if the command was successful, and non-zero if there was any error. `, DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(_ *cobra.Command, args []string) error { return runGenerate(genOpts, args) }, } diff --git a/cmd/restic/cmd_key_list.go b/cmd/restic/cmd_key_list.go index 517b7c84b..2b3574281 100644 --- a/cmd/restic/cmd_key_list.go +++ b/cmd/restic/cmd_key_list.go @@ -69,7 +69,7 @@ func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions var m sync.Mutex var keys []keyInfo - err := restic.ParallelList(ctx, s, restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, size int64) error { + err := restic.ParallelList(ctx, s, restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, _ int64) error { k, err := repository.LoadKey(ctx, s, id) if err != nil { Warnf("LoadKey() failed: %v\n", err) diff --git a/cmd/restic/cmd_list.go b/cmd/restic/cmd_list.go index 8be99234f..becad7f0d 100644 --- a/cmd/restic/cmd_list.go +++ b/cmd/restic/cmd_list.go @@ -63,7 +63,7 @@ func runList(ctx context.Context, gopts GlobalOptions, args []string) error { case "locks": t = restic.LockFile case "blobs": - return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { + return index.ForAllIndexes(ctx, repo, repo, func(_ restic.ID, idx *index.Index, _ bool, err error) error { if err != nil { return err } @@ -76,7 +76,7 @@ func runList(ctx context.Context, gopts GlobalOptions, args []string) error { return errors.Fatal("invalid type") } - return repo.List(ctx, t, func(id restic.ID, size int64) error { + return repo.List(ctx, t, func(id restic.ID, _ int64) error { Printf("%s\n", id) return nil }) diff --git a/cmd/restic/cmd_options.go b/cmd/restic/cmd_options.go index 471319dfb..85e062220 100644 --- a/cmd/restic/cmd_options.go +++ b/cmd/restic/cmd_options.go @@ -21,7 +21,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er `, Hidden: true, DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, _ []string) { fmt.Printf("All Extended Options:\n") var maxLen int for _, opt := range options.List() { diff --git a/cmd/restic/cmd_prune.go b/cmd/restic/cmd_prune.go index efd8f6e3a..1b9352ea7 100644 --- a/cmd/restic/cmd_prune.go +++ b/cmd/restic/cmd_prune.go @@ -37,7 +37,7 @@ EXIT STATUS Exit status is 0 if the command was successful, and non-zero if there was any error. `, DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { return runPrune(cmd.Context(), pruneOptions, globalOptions) }, } @@ -101,7 +101,7 @@ func verifyPruneOptions(opts *PruneOptions) error { // parse MaxUnused either as unlimited, a percentage, or an absolute number of bytes switch { case maxUnused == "unlimited": - opts.maxUnusedBytes = func(used uint64) uint64 { + opts.maxUnusedBytes = func(_ uint64) uint64 { return math.MaxUint64 } @@ -130,7 +130,7 @@ func verifyPruneOptions(opts *PruneOptions) error { return errors.Fatalf("invalid number of bytes %q for --max-unused: %v", opts.MaxUnused, err) } - opts.maxUnusedBytes = func(used uint64) uint64 { + opts.maxUnusedBytes = func(_ uint64) uint64 { return uint64(size) } } @@ -798,7 +798,7 @@ func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Rep DeleteProgress: func() *progress.Counter { return newProgressMax(!gopts.Quiet, 0, "old indexes deleted") }, - DeleteReport: func(id restic.ID, err error) { + DeleteReport: func(id restic.ID, _ error) { if gopts.verbosity > 2 { Verbosef("removed index %v\n", id.String()) } diff --git a/cmd/restic/cmd_recover.go b/cmd/restic/cmd_recover.go index 499abdf80..b97a7582b 100644 --- a/cmd/restic/cmd_recover.go +++ b/cmd/restic/cmd_recover.go @@ -25,7 +25,7 @@ EXIT STATUS Exit status is 0 if the command was successful, and non-zero if there was any error. `, DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { return runRecover(cmd.Context(), globalOptions) }, } @@ -91,7 +91,7 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error { bar.Done() Verbosef("load snapshots\n") - err = restic.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(id restic.ID, sn *restic.Snapshot, err error) error { + err = restic.ForAllSnapshots(ctx, snapshotLister, repo, nil, func(_ restic.ID, sn *restic.Snapshot, _ error) error { trees[*sn.Tree] = true return nil }) diff --git a/cmd/restic/cmd_repair_index.go b/cmd/restic/cmd_repair_index.go index fc5506b34..ea36f02f6 100644 --- a/cmd/restic/cmd_repair_index.go +++ b/cmd/restic/cmd_repair_index.go @@ -24,7 +24,7 @@ EXIT STATUS Exit status is 0 if the command was successful, and non-zero if there was any error. `, DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { return runRebuildIndex(cmd.Context(), repairIndexOptions, globalOptions) }, } @@ -78,7 +78,7 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti if opts.ReadAllPacks { // get list of old index files but start with empty index - err := repo.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error { + err := repo.List(ctx, restic.IndexFile, func(id restic.ID, _ int64) error { obsoleteIndexes = append(obsoleteIndexes, id) return nil }) @@ -88,7 +88,7 @@ func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOpti } else { Verbosef("loading indexes...\n") mi := index.NewMasterIndex() - err := index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { + err := index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, _ bool, err error) error { if err != nil { Warnf("removing invalid index %v: %v\n", id, err) obsoleteIndexes = append(obsoleteIndexes, id) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index 19e457b1f..cc3d0eb85 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -125,7 +125,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt node.Size = newSize return node }, - RewriteFailedTree: func(nodeID restic.ID, path string, _ error) (restic.ID, error) { + RewriteFailedTree: func(_ restic.ID, path string, _ error) (restic.ID, error) { if path == "/" { Verbosef(" dir %q: not readable\n", path) // remove snapshots with invalid root node diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 1208d30eb..37d304672 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -181,7 +181,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, excludePatterns := filter.ParsePatterns(opts.Exclude) insensitiveExcludePatterns := filter.ParsePatterns(opts.InsensitiveExclude) - selectExcludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { + selectExcludeFilter := func(item string, _ string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { matched, err := filter.List(excludePatterns, item) if err != nil { msg.E("error for exclude pattern: %v", err) @@ -204,7 +204,7 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, includePatterns := filter.ParsePatterns(opts.Include) insensitiveIncludePatterns := filter.ParsePatterns(opts.InsensitiveInclude) - selectIncludeFilter := func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { + selectIncludeFilter := func(item string, _ string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) { matched, childMayMatch, err := filter.ListWithChild(includePatterns, item) if err != nil { msg.E("error for include pattern: %v", err) diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index 8dad492af..62624e75c 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -147,7 +147,7 @@ func rewriteSnapshot(ctx context.Context, repo *repository.Repository, sn *resti return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) } } else { - filter = func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { + filter = func(_ context.Context, sn *restic.Snapshot) (restic.ID, error) { return *sn.Tree, nil } } diff --git a/cmd/restic/cmd_stats.go b/cmd/restic/cmd_stats.go index 1bece21d0..d3078a419 100644 --- a/cmd/restic/cmd_stats.go +++ b/cmd/restic/cmd_stats.go @@ -367,7 +367,7 @@ func statsDebug(ctx context.Context, repo restic.Repository) error { func statsDebugFileType(ctx context.Context, repo restic.Lister, tpe restic.FileType) (*sizeHistogram, error) { hist := newSizeHistogram(2 * repository.MaxPackSize) - err := repo.List(ctx, tpe, func(id restic.ID, size int64) error { + err := repo.List(ctx, tpe, func(_ restic.ID, size int64) error { hist.Add(uint64(size)) return nil }) diff --git a/cmd/restic/cmd_unlock.go b/cmd/restic/cmd_unlock.go index 7b449d949..6893f3365 100644 --- a/cmd/restic/cmd_unlock.go +++ b/cmd/restic/cmd_unlock.go @@ -19,7 +19,7 @@ EXIT STATUS Exit status is 0 if the command was successful, and non-zero if there was any error. `, DisableAutoGenTag: true, - RunE: func(cmd *cobra.Command, args []string) error { + RunE: func(cmd *cobra.Command, _ []string) error { return runUnlock(cmd.Context(), unlockOptions, globalOptions) }, } diff --git a/cmd/restic/cmd_version.go b/cmd/restic/cmd_version.go index 73469750f..e3f9b3010 100644 --- a/cmd/restic/cmd_version.go +++ b/cmd/restic/cmd_version.go @@ -21,7 +21,7 @@ EXIT STATUS Exit status is 0 if the command was successful, and non-zero if there was any error. `, DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + Run: func(_ *cobra.Command, _ []string) { if globalOptions.JSON { type jsonVersion struct { Version string `json:"version"` diff --git a/cmd/restic/main.go b/cmd/restic/main.go index 4595e8161..b31ce1bb4 100644 --- a/cmd/restic/main.go +++ b/cmd/restic/main.go @@ -37,7 +37,7 @@ The full documentation can be found at https://restic.readthedocs.io/ . SilenceUsage: true, DisableAutoGenTag: true, - PersistentPreRunE: func(c *cobra.Command, args []string) error { + PersistentPreRunE: func(c *cobra.Command, _ []string) error { // set verbosity, default is one globalOptions.verbosity = 1 if globalOptions.Quiet && globalOptions.Verbose > 0 { diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index f2c481b32..77ddba7c4 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -147,8 +147,8 @@ func (o Options) ApplyDefaults() Options { func New(repo restic.Repository, fs fs.FS, opts Options) *Archiver { arch := &Archiver{ Repo: repo, - SelectByName: func(item string) bool { return true }, - Select: func(item string, fi os.FileInfo) bool { return true }, + SelectByName: func(_ string) bool { return true }, + Select: func(_ string, _ os.FileInfo) bool { return true }, FS: fs, Options: opts.ApplyDefaults(), @@ -762,7 +762,7 @@ func (arch *Archiver) Snapshot(ctx context.Context, targets []string, opts Snaps arch.runWorkers(wgCtx, wg) debug.Log("starting snapshot") - fn, nodeCount, err := arch.SaveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(n *restic.Node, is ItemStats) { + fn, nodeCount, err := arch.SaveTree(wgCtx, "/", atree, arch.loadParentTree(wgCtx, opts.ParentSnapshot), func(_ *restic.Node, is ItemStats) { arch.CompleteItem("/", nil, nil, is, time.Since(start)) }) if err != nil { diff --git a/internal/archiver/scanner.go b/internal/archiver/scanner.go index 6ce2a4700..cc419b19e 100644 --- a/internal/archiver/scanner.go +++ b/internal/archiver/scanner.go @@ -25,10 +25,10 @@ type Scanner struct { func NewScanner(fs fs.FS) *Scanner { return &Scanner{ FS: fs, - SelectByName: func(item string) bool { return true }, - Select: func(item string, fi os.FileInfo) bool { return true }, - Error: func(item string, err error) error { return err }, - Result: func(item string, s ScanStats) {}, + SelectByName: func(_ string) bool { return true }, + Select: func(_ string, _ os.FileInfo) bool { return true }, + Error: func(_ string, err error) error { return err }, + Result: func(_ string, _ ScanStats) {}, } } diff --git a/internal/backend/mem/mem_backend.go b/internal/backend/mem/mem_backend.go index 2698a8275..eea5b060e 100644 --- a/internal/backend/mem/mem_backend.go +++ b/internal/backend/mem/mem_backend.go @@ -28,7 +28,7 @@ func NewFactory() location.Factory { return location.NewHTTPBackendFactory[struct{}, *MemoryBackend]( "mem", - func(s string) (*struct{}, error) { + func(_ string) (*struct{}, error) { return &struct{}{}, nil }, location.NoPassword, diff --git a/internal/backend/rclone/backend.go b/internal/backend/rclone/backend.go index 416162364..25082598f 100644 --- a/internal/backend/rclone/backend.go +++ b/internal/backend/rclone/backend.go @@ -183,7 +183,7 @@ func newBackend(ctx context.Context, cfg Config, lim limiter.Limiter) (*Backend, dialCount := 0 tr := &http2.Transport{ AllowHTTP: true, // this is not really HTTP, just stdin/stdout - DialTLS: func(network, address string, cfg *tls.Config) (net.Conn, error) { + DialTLS: func(network, address string, _ *tls.Config) (net.Conn, error) { debug.Log("new connection requested, %v %v", network, address) if dialCount > 0 { // the connection to the child process is already closed diff --git a/internal/checker/checker.go b/internal/checker/checker.go index df865cb41..1e14a9e53 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -134,7 +134,7 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e if p != nil { var numIndexFiles uint64 - err := indexList.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error { + err := indexList.List(ctx, restic.IndexFile, func(_ restic.ID, _ int64) error { numIndexFiles++ return nil }) diff --git a/internal/fs/fs_track.go b/internal/fs/fs_track.go index 319fbfaff..0c65a8564 100644 --- a/internal/fs/fs_track.go +++ b/internal/fs/fs_track.go @@ -41,7 +41,7 @@ type trackFile struct { func newTrackFile(stack []byte, filename string, file File) *trackFile { f := &trackFile{file} - runtime.SetFinalizer(f, func(f *trackFile) { + runtime.SetFinalizer(f, func(_ *trackFile) { fmt.Fprintf(os.Stderr, "file %s not closed\n\nStacktrack:\n%s\n", filename, stack) panic("file " + filename + " not closed") }) diff --git a/internal/fuse/snapshots_dirstruct.go b/internal/fuse/snapshots_dirstruct.go index d40ae6298..049319c6f 100644 --- a/internal/fuse/snapshots_dirstruct.go +++ b/internal/fuse/snapshots_dirstruct.go @@ -295,7 +295,7 @@ func (d *SnapshotsDirStructure) updateSnapshots(ctx context.Context) error { } var snapshots restic.Snapshots - err := d.root.cfg.Filter.FindAll(ctx, d.root.repo, d.root.repo, nil, func(id string, sn *restic.Snapshot, err error) error { + err := d.root.cfg.Filter.FindAll(ctx, d.root.repo, d.root.repo, nil, func(_ string, sn *restic.Snapshot, _ error) error { if sn != nil { snapshots = append(snapshots, sn) } diff --git a/internal/index/index_parallel.go b/internal/index/index_parallel.go index 3c16d049b..d51d5930f 100644 --- a/internal/index/index_parallel.go +++ b/internal/index/index_parallel.go @@ -19,7 +19,7 @@ func ForAllIndexes(ctx context.Context, lister restic.Lister, repo restic.Lister workerCount := repo.Connections() + uint(runtime.GOMAXPROCS(0)) var m sync.Mutex - return restic.ParallelList(ctx, lister, restic.IndexFile, workerCount, func(ctx context.Context, id restic.ID, size int64) error { + return restic.ParallelList(ctx, lister, restic.IndexFile, workerCount, func(ctx context.Context, id restic.ID, _ int64) error { var err error var idx *Index oldFormat := false diff --git a/internal/repository/key.go b/internal/repository/key.go index 5f7a93297..d9f8d8e17 100644 --- a/internal/repository/key.go +++ b/internal/repository/key.go @@ -136,7 +136,7 @@ func SearchKey(ctx context.Context, s *Repository, password string, maxKeys int, defer cancel() // try at most maxKeys keys in repo - err = s.List(listCtx, restic.KeyFile, func(id restic.ID, size int64) error { + err = s.List(listCtx, restic.KeyFile, func(id restic.ID, _ int64) error { checked++ if maxKeys > 0 && checked > maxKeys { return ErrMaxKeysReached diff --git a/internal/repository/repair_pack.go b/internal/repository/repair_pack.go index 64279e727..2e0368899 100644 --- a/internal/repository/repair_pack.go +++ b/internal/repository/repair_pack.go @@ -68,7 +68,7 @@ func RepairPacks(ctx context.Context, repo restic.Repository, ids restic.IDSet, DeleteProgress: func() *progress.Counter { return printer.NewCounter("old indexes deleted") }, - DeleteReport: func(id restic.ID, err error) { + DeleteReport: func(id restic.ID, _ error) { printer.VV("removed index %v", id.String()) }, }) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index a20f71ab1..9a4494c4d 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -651,7 +651,7 @@ func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error { if p != nil { var numIndexFiles uint64 - err := indexList.List(ctx, restic.IndexFile, func(id restic.ID, size int64) error { + err := indexList.List(ctx, restic.IndexFile, func(_ restic.ID, _ int64) error { numIndexFiles++ return nil }) @@ -662,7 +662,7 @@ func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error { defer p.Done() } - err = index.ForAllIndexes(ctx, indexList, r, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error { + err = index.ForAllIndexes(ctx, indexList, r, func(_ restic.ID, idx *index.Index, _ bool, err error) error { if err != nil { return err } diff --git a/internal/restic/backend_find.go b/internal/restic/backend_find.go index a6eacabd0..2f00595c4 100644 --- a/internal/restic/backend_find.go +++ b/internal/restic/backend_find.go @@ -30,7 +30,7 @@ func Find(ctx context.Context, be Lister, t FileType, prefix string) (ID, error) ctx, cancel := context.WithCancel(ctx) defer cancel() - err := be.List(ctx, t, func(id ID, size int64) error { + err := be.List(ctx, t, func(id ID, _ int64) error { name := id.String() if len(name) >= len(prefix) && prefix == name[:len(prefix)] { if match.IsNull() { diff --git a/internal/restic/lock.go b/internal/restic/lock.go index d98f8ec94..182a3442d 100644 --- a/internal/restic/lock.go +++ b/internal/restic/lock.go @@ -341,7 +341,7 @@ func (l *Lock) checkExistence(ctx context.Context) (bool, error) { exists := false - err := l.repo.List(ctx, LockFile, func(id ID, size int64) error { + err := l.repo.List(ctx, LockFile, func(id ID, _ int64) error { if id.Equal(*l.lockID) { exists = true } @@ -415,7 +415,7 @@ func RemoveStaleLocks(ctx context.Context, repo Repository) (uint, error) { // RemoveAllLocks removes all locks forcefully. func RemoveAllLocks(ctx context.Context, repo Repository) (uint, error) { var processed uint32 - err := ParallelList(ctx, repo, LockFile, repo.Connections(), func(ctx context.Context, id ID, size int64) error { + err := ParallelList(ctx, repo, LockFile, repo.Connections(), func(ctx context.Context, id ID, _ int64) error { err := repo.Backend().Remove(ctx, backend.Handle{Type: LockFile, Name: id.String()}) if err == nil { atomic.AddUint32(&processed, 1) diff --git a/internal/restic/snapshot.go b/internal/restic/snapshot.go index 88171a646..8cf651d96 100644 --- a/internal/restic/snapshot.go +++ b/internal/restic/snapshot.go @@ -83,7 +83,7 @@ func ForAllSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked, excl var m sync.Mutex // For most snapshots decoding is nearly for free, thus just assume were only limited by IO - return ParallelList(ctx, be, SnapshotFile, loader.Connections(), func(ctx context.Context, id ID, size int64) error { + return ParallelList(ctx, be, SnapshotFile, loader.Connections(), func(ctx context.Context, id ID, _ int64) error { if excludeIDs.Has(id) { return nil } diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 2ce1ee98e..3f4fb32e3 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -27,7 +27,7 @@ type Restorer struct { SelectFilter func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) } -var restorerAbortOnAllErrors = func(location string, err error) error { return err } +var restorerAbortOnAllErrors = func(_ string, err error) error { return err } // NewRestorer creates a restorer preloaded with the content from the snapshot id. func NewRestorer(repo restic.Repository, sn *restic.Snapshot, sparse bool, @@ -239,7 +239,7 @@ func (res *Restorer) RestoreTo(ctx context.Context, dst string) error { // first tree pass: create directories and collect all files to restore _, err = res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{ - enterDir: func(node *restic.Node, target, location string) error { + enterDir: func(_ *restic.Node, target, location string) error { debug.Log("first pass, enterDir: mkdir %q, leaveDir should restore metadata", location) if res.progress != nil { res.progress.AddFile(0) @@ -366,7 +366,7 @@ func (res *Restorer) VerifyFiles(ctx context.Context, dst string) (int, error) { defer close(work) _, err := res.traverseTree(ctx, dst, string(filepath.Separator), *res.sn.Tree, treeVisitor{ - visitNode: func(node *restic.Node, target, location string) error { + visitNode: func(node *restic.Node, target, _ string) error { if node.Type != "file" { return nil } diff --git a/internal/ui/backup/progress.go b/internal/ui/backup/progress.go index 4362a8c83..da0d401a3 100644 --- a/internal/ui/backup/progress.go +++ b/internal/ui/backup/progress.go @@ -63,7 +63,7 @@ func NewProgress(printer ProgressPrinter, interval time.Duration) *Progress { printer: printer, estimator: *newRateEstimator(time.Now()), } - p.Updater = *progress.NewUpdater(interval, func(runtime time.Duration, final bool) { + p.Updater = *progress.NewUpdater(interval, func(_ time.Duration, final bool) { if final { p.printer.Reset() } else { diff --git a/internal/walker/rewriter.go b/internal/walker/rewriter.go index 649857032..6d283a625 100644 --- a/internal/walker/rewriter.go +++ b/internal/walker/rewriter.go @@ -39,13 +39,13 @@ func NewTreeRewriter(opts RewriteOpts) *TreeRewriter { } // setup default implementations if rw.opts.RewriteNode == nil { - rw.opts.RewriteNode = func(node *restic.Node, path string) *restic.Node { + rw.opts.RewriteNode = func(node *restic.Node, _ string) *restic.Node { return node } } if rw.opts.RewriteFailedTree == nil { // fail with error by default - rw.opts.RewriteFailedTree = func(nodeID restic.ID, path string, err error) (restic.ID, error) { + rw.opts.RewriteFailedTree = func(_ restic.ID, _ string, err error) (restic.ID, error) { return restic.ID{}, err } } From 2ab18a92e6b00714b539439ea15656471bcb6b78 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 10 Feb 2024 23:42:34 +0100 Subject: [PATCH 134/158] CI: keep tests for Go 1.19 --- .github/workflows/tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cc599491c..091d42e8a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -54,6 +54,11 @@ jobs: os: ubuntu-latest test_fuse: true + - job_name: Linux + go: 1.19.x + os: ubuntu-latest + test_fuse: true + name: ${{ matrix.job_name }} Go ${{ matrix.go }} runs-on: ${{ matrix.os }} From ee1ff3c1d0f6d596ef559e49f9e11a1c53640b0b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 11 Feb 2024 22:26:13 +0100 Subject: [PATCH 135/158] backup: improve error message if exclude file cannot be loaded --- cmd/restic/exclude.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/exclude.go b/cmd/restic/exclude.go index 095944610..d9bb63aeb 100644 --- a/cmd/restic/exclude.go +++ b/cmd/restic/exclude.go @@ -426,7 +426,7 @@ func readExcludePatternsFromFiles(excludeFiles []string) ([]string, error) { return scanner.Err() }() if err != nil { - return nil, err + return nil, fmt.Errorf("failed to read excludes from file %q: %w", filename, err) } } return excludes, nil From 5b5d506472876e3a5916d9943a911db2c5faa887 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 10 Feb 2024 23:41:11 +0100 Subject: [PATCH 136/158] backup: report files whose chunks failed to upload --- internal/archiver/blob_saver.go | 8 +++++--- internal/archiver/blob_saver_test.go | 11 ++++++----- internal/archiver/file_saver.go | 4 ++-- internal/archiver/file_saver_test.go | 2 +- internal/archiver/tree_saver.go | 6 +++--- internal/archiver/tree_saver_test.go | 2 +- 6 files changed, 18 insertions(+), 15 deletions(-) diff --git a/internal/archiver/blob_saver.go b/internal/archiver/blob_saver.go index ae4879ff4..d4347a169 100644 --- a/internal/archiver/blob_saver.go +++ b/internal/archiver/blob_saver.go @@ -2,6 +2,7 @@ package archiver import ( "context" + "fmt" "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/restic" @@ -43,9 +44,9 @@ func (s *BlobSaver) TriggerShutdown() { // Save stores a blob in the repo. It checks the index and the known blobs // before saving anything. It takes ownership of the buffer passed in. -func (s *BlobSaver) Save(ctx context.Context, t restic.BlobType, buf *Buffer, cb func(res SaveBlobResponse)) { +func (s *BlobSaver) Save(ctx context.Context, t restic.BlobType, buf *Buffer, filename string, cb func(res SaveBlobResponse)) { select { - case s.ch <- saveBlobJob{BlobType: t, buf: buf, cb: cb}: + case s.ch <- saveBlobJob{BlobType: t, buf: buf, fn: filename, cb: cb}: case <-ctx.Done(): debug.Log("not sending job, context is cancelled") } @@ -54,6 +55,7 @@ func (s *BlobSaver) Save(ctx context.Context, t restic.BlobType, buf *Buffer, cb type saveBlobJob struct { restic.BlobType buf *Buffer + fn string cb func(res SaveBlobResponse) } @@ -95,7 +97,7 @@ func (s *BlobSaver) worker(ctx context.Context, jobs <-chan saveBlobJob) error { res, err := s.saveBlob(ctx, job.BlobType, job.buf.Data) if err != nil { debug.Log("saveBlob returned error, exiting: %v", err) - return err + return fmt.Errorf("failed to save blob from file %q: %w", job.fn, err) } job.cb(res) job.buf.Release() diff --git a/internal/archiver/blob_saver_test.go b/internal/archiver/blob_saver_test.go index 1996c35b8..180f95b3d 100644 --- a/internal/archiver/blob_saver_test.go +++ b/internal/archiver/blob_saver_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "runtime" + "strings" "sync" "sync/atomic" "testing" @@ -11,6 +12,7 @@ import ( "github.com/restic/restic/internal/errors" "github.com/restic/restic/internal/index" "github.com/restic/restic/internal/restic" + rtest "github.com/restic/restic/internal/test" "golang.org/x/sync/errgroup" ) @@ -57,7 +59,7 @@ func TestBlobSaver(t *testing.T) { lock.Lock() results = append(results, SaveBlobResponse{}) lock.Unlock() - b.Save(ctx, restic.DataBlob, buf, func(res SaveBlobResponse) { + b.Save(ctx, restic.DataBlob, buf, "file", func(res SaveBlobResponse) { lock.Lock() results[idx] = res lock.Unlock() @@ -106,7 +108,7 @@ func TestBlobSaverError(t *testing.T) { for i := 0; i < test.blobs; i++ { buf := &Buffer{Data: []byte(fmt.Sprintf("foo%d", i))} - b.Save(ctx, restic.DataBlob, buf, func(res SaveBlobResponse) {}) + b.Save(ctx, restic.DataBlob, buf, "errfile", func(res SaveBlobResponse) {}) } b.TriggerShutdown() @@ -116,9 +118,8 @@ func TestBlobSaverError(t *testing.T) { t.Errorf("expected error not found") } - if err != errTest { - t.Fatalf("unexpected error found: %v", err) - } + rtest.Assert(t, errors.Is(err, errTest), "unexpected error %v", err) + rtest.Assert(t, strings.Contains(err.Error(), "errfile"), "expected error to contain 'errfile' got: %v", err) }) } } diff --git a/internal/archiver/file_saver.go b/internal/archiver/file_saver.go index 724f5e620..7f11bff8a 100644 --- a/internal/archiver/file_saver.go +++ b/internal/archiver/file_saver.go @@ -16,7 +16,7 @@ import ( ) // SaveBlobFn saves a blob to a repo. -type SaveBlobFn func(context.Context, restic.BlobType, *Buffer, func(res SaveBlobResponse)) +type SaveBlobFn func(context.Context, restic.BlobType, *Buffer, string, func(res SaveBlobResponse)) // FileSaver concurrently saves incoming files to the repo. type FileSaver struct { @@ -205,7 +205,7 @@ func (s *FileSaver) saveFile(ctx context.Context, chnker *chunker.Chunker, snPat node.Content = append(node.Content, restic.ID{}) lock.Unlock() - s.saveBlob(ctx, restic.DataBlob, buf, func(sbr SaveBlobResponse) { + s.saveBlob(ctx, restic.DataBlob, buf, target, func(sbr SaveBlobResponse) { lock.Lock() if !sbr.known { fnr.stats.DataBlobs++ diff --git a/internal/archiver/file_saver_test.go b/internal/archiver/file_saver_test.go index b088eeeed..ced9d796e 100644 --- a/internal/archiver/file_saver_test.go +++ b/internal/archiver/file_saver_test.go @@ -33,7 +33,7 @@ func createTestFiles(t testing.TB, num int) (files []string) { func startFileSaver(ctx context.Context, t testing.TB) (*FileSaver, context.Context, *errgroup.Group) { wg, ctx := errgroup.WithContext(ctx) - saveBlob := func(ctx context.Context, tpe restic.BlobType, buf *Buffer, cb func(SaveBlobResponse)) { + saveBlob := func(ctx context.Context, tpe restic.BlobType, buf *Buffer, _ string, cb func(SaveBlobResponse)) { cb(SaveBlobResponse{ id: restic.Hash(buf.Data), length: len(buf.Data), diff --git a/internal/archiver/tree_saver.go b/internal/archiver/tree_saver.go index a7dae3873..eae524a78 100644 --- a/internal/archiver/tree_saver.go +++ b/internal/archiver/tree_saver.go @@ -11,7 +11,7 @@ import ( // TreeSaver concurrently saves incoming trees to the repo. type TreeSaver struct { - saveBlob func(ctx context.Context, t restic.BlobType, buf *Buffer, cb func(res SaveBlobResponse)) + saveBlob SaveBlobFn errFn ErrorFunc ch chan<- saveTreeJob @@ -19,7 +19,7 @@ type TreeSaver struct { // NewTreeSaver returns a new tree saver. A worker pool with treeWorkers is // started, it is stopped when ctx is cancelled. -func NewTreeSaver(ctx context.Context, wg *errgroup.Group, treeWorkers uint, saveBlob func(ctx context.Context, t restic.BlobType, buf *Buffer, cb func(res SaveBlobResponse)), errFn ErrorFunc) *TreeSaver { +func NewTreeSaver(ctx context.Context, wg *errgroup.Group, treeWorkers uint, saveBlob SaveBlobFn, errFn ErrorFunc) *TreeSaver { ch := make(chan saveTreeJob) s := &TreeSaver{ @@ -126,7 +126,7 @@ func (s *TreeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, I b := &Buffer{Data: buf} ch := make(chan SaveBlobResponse, 1) - s.saveBlob(ctx, restic.TreeBlob, b, func(res SaveBlobResponse) { + s.saveBlob(ctx, restic.TreeBlob, b, job.target, func(res SaveBlobResponse) { ch <- res }) diff --git a/internal/archiver/tree_saver_test.go b/internal/archiver/tree_saver_test.go index 5de4375d6..47a3f3842 100644 --- a/internal/archiver/tree_saver_test.go +++ b/internal/archiver/tree_saver_test.go @@ -12,7 +12,7 @@ import ( "golang.org/x/sync/errgroup" ) -func treeSaveHelper(_ context.Context, _ restic.BlobType, buf *Buffer, cb func(res SaveBlobResponse)) { +func treeSaveHelper(_ context.Context, _ restic.BlobType, buf *Buffer, _ string, cb func(res SaveBlobResponse)) { cb(SaveBlobResponse{ id: restic.NewRandomID(), known: false, From 22a3cea1b30db58deaaabd430a286f0141850514 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 27 Jan 2024 18:59:32 +0100 Subject: [PATCH 137/158] checker: wrap all pack errors in ErrPackData --- internal/checker/checker.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 1e14a9e53..df126f539 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -521,7 +521,7 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []r debug.Log("checking pack %v", id.String()) if len(blobs) == 0 { - return errors.Errorf("pack %v is empty or not indexed", id) + return &ErrPackData{PackID: id, errs: []error{errors.New("pack is empty or not indexed")}} } // sanity check blobs in index @@ -542,7 +542,7 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []r var errs []error if nonContinuousPack { debug.Log("Index for pack contains gaps / overlaps, blobs: %v", blobs) - errs = append(errs, errors.New("Index for pack contains gaps / overlapping blobs")) + errs = append(errs, errors.New("index for pack contains gaps / overlapping blobs")) } // calculate hash on-the-fly while reading the pack and capture pack header @@ -591,21 +591,21 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []r if err != nil { // failed to load the pack file, return as further checks cannot succeed anyways debug.Log(" error streaming pack: %v", err) - return errors.Errorf("pack %v failed to download: %v", id, err) + return &ErrPackData{PackID: id, errs: append(errs, errors.Errorf("download error: %w", err))} } if !hash.Equal(id) { - debug.Log("Pack ID does not match, want %v, got %v", id, hash) - return errors.Errorf("Pack ID does not match, want %v, got %v", id, hash) + debug.Log("pack ID does not match, want %v, got %v", id, hash) + return &ErrPackData{PackID: id, errs: append(errs, errors.Errorf("unexpected pack id %v", hash))} } blobs, hdrSize, err := pack.List(r.Key(), bytes.NewReader(hdrBuf), int64(len(hdrBuf))) if err != nil { - return err + return &ErrPackData{PackID: id, errs: append(errs, err)} } if uint32(idxHdrSize) != hdrSize { debug.Log("Pack header size does not match, want %v, got %v", idxHdrSize, hdrSize) - errs = append(errs, errors.Errorf("Pack header size does not match, want %v, got %v", idxHdrSize, hdrSize)) + errs = append(errs, errors.Errorf("pack header size does not match, want %v, got %v", idxHdrSize, hdrSize)) } idx := r.Index() @@ -619,7 +619,7 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []r } } if !idxHas { - errs = append(errs, errors.Errorf("Blob %v is not contained in index or position is incorrect", blob.ID)) + errs = append(errs, errors.Errorf("blob %v is not contained in index or position is incorrect", blob.ID)) continue } } From 772e3416d1204bd43b539e9ffcfeb5c5602f02db Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 27 Jan 2024 18:59:54 +0100 Subject: [PATCH 138/158] repair pack: drop feature flag --- cmd/restic/cmd_check.go | 2 +- cmd/restic/cmd_repair_packs.go | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index 22f462d75..21c9cc899 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -349,7 +349,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args for _, id := range salvagePacks { strIDs = append(strIDs, id.String()) } - Warnf("RESTIC_FEATURES=repair-packs-v1 restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIDs, " ")) + Warnf("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIDs, " ")) Warnf("Corrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n") } } diff --git a/cmd/restic/cmd_repair_packs.go b/cmd/restic/cmd_repair_packs.go index 04b06c33b..521b5859f 100644 --- a/cmd/restic/cmd_repair_packs.go +++ b/cmd/restic/cmd_repair_packs.go @@ -40,13 +40,6 @@ func init() { } func runRepairPacks(ctx context.Context, gopts GlobalOptions, term *termstatus.Terminal, args []string) error { - // FIXME discuss and add proper feature flag mechanism - flag, _ := os.LookupEnv("RESTIC_FEATURES") - if flag != "repair-packs-v1" { - return errors.Fatal("This command is experimental and may change/be removed without notice between restic versions. " + - "Set the environment variable 'RESTIC_FEATURES=repair-packs-v1' to enable it.") - } - ids := restic.NewIDSet() for _, arg := range args { id, err := restic.ParseID(arg) From 544fe38786eeab7aa4077a7c692d2f5e7d994ae8 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 27 Jan 2024 19:00:23 +0100 Subject: [PATCH 139/158] check: suggest repair pack for all damaged packs --- cmd/restic/cmd_check.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index 21c9cc899..8302c72bc 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -336,9 +336,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args errorsFound = true Warnf("%v\n", err) if err, ok := err.(*checker.ErrPackData); ok { - if strings.Contains(err.Error(), "wrong data returned, hash is") { - salvagePacks = append(salvagePacks, err.PackID) - } + salvagePacks = append(salvagePacks, err.PackID) } } p.Done() From 6397615fbbca6425e71f4927e6b5c2188f363ed5 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 27 Jan 2024 19:04:45 +0100 Subject: [PATCH 140/158] check: document that check will show repair pack instructions --- doc/077_troubleshooting.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/077_troubleshooting.rst b/doc/077_troubleshooting.rst index 6a9a6ee15..512b41b0d 100644 --- a/doc/077_troubleshooting.rst +++ b/doc/077_troubleshooting.rst @@ -76,6 +76,8 @@ Similarly, if a repository is repeatedly damaged, please open an `issue on Githu somewhere. Please include the check output and additional information that might help locate the problem. +If ``check`` detects damaged pack files, it will show instructions on how to repair +them. Please follow the steps from the next section first. 2. Backup the repository ************************ From 4073299a7cba6874cfe30467cea86405027d8c42 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 4 Feb 2024 17:22:09 +0100 Subject: [PATCH 141/158] check: fix missing error if blob is invalid --- internal/checker/checker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index df126f539..0fdd3d942 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -563,8 +563,8 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []r } debug.Log(" check blob %v: %v", val.Handle.ID, val.Handle) if val.Err != nil { - debug.Log(" error verifying blob %v: %v", val.Handle.ID, err) - errs = append(errs, errors.Errorf("blob %v: %v", val.Handle.ID, err)) + debug.Log(" error verifying blob %v: %v", val.Handle.ID, val.Err) + errs = append(errs, errors.Errorf("blob %v: %v", val.Handle.ID, val.Err)) } } From ed4a4f8748320a4aab2591c6d478fa442637f7de Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 10 Feb 2024 20:27:17 +0100 Subject: [PATCH 142/158] check: exclude inaccessible files from the repair pack suggestion --- internal/checker/checker.go | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/internal/checker/checker.go b/internal/checker/checker.go index 0fdd3d942..28f55ce3a 100644 --- a/internal/checker/checker.go +++ b/internal/checker/checker.go @@ -516,6 +516,14 @@ func (c *Checker) GetPacks() map[restic.ID]int64 { return c.packs } +type partialReadError struct { + err error +} + +func (e *partialReadError) Error() string { + return e.err.Error() +} + // checkPack reads a pack and checks the integrity of all blobs. func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []restic.Blob, size int64, bufRd *bufio.Reader, dec *zstd.Decoder) error { debug.Log("checking pack %v", id.String()) @@ -559,7 +567,7 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []r if err == repository.ErrPackEOF { break } else if err != nil { - return err + return &partialReadError{err} } debug.Log(" check blob %v: %v", val.Handle.ID, val.Handle) if val.Err != nil { @@ -574,7 +582,7 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []r if minHdrStart > curPos { _, err := bufRd.Discard(minHdrStart - curPos) if err != nil { - return err + return &partialReadError{err} } } @@ -582,16 +590,24 @@ func checkPack(ctx context.Context, r restic.Repository, id restic.ID, blobs []r var err error hdrBuf, err = io.ReadAll(bufRd) if err != nil { - return err + return &partialReadError{err} } hash = restic.IDFromHash(hrd.Sum(nil)) return nil }) if err != nil { + var e *partialReadError + isPartialReadError := errors.As(err, &e) // failed to load the pack file, return as further checks cannot succeed anyways - debug.Log(" error streaming pack: %v", err) - return &ErrPackData{PackID: id, errs: append(errs, errors.Errorf("download error: %w", err))} + debug.Log(" error streaming pack (partial %v): %v", isPartialReadError, err) + if isPartialReadError { + return &ErrPackData{PackID: id, errs: append(errs, errors.Errorf("partial download error: %w", err))} + } + + // The check command suggests to repair files for which a `ErrPackData` is returned. However, this file + // completely failed to download such that there's no point in repairing anything. + return errors.Errorf("download error: %w", err) } if !hash.Equal(id) { debug.Log("pack ID does not match, want %v, got %v", id, hash) From 527a3ff2b218902d9d4e3c6eab7e67b8b2edea74 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 10 Feb 2024 20:27:47 +0100 Subject: [PATCH 143/158] check: link to troubleshooting guide --- cmd/restic/cmd_check.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/cmd_check.go b/cmd/restic/cmd_check.go index 8302c72bc..990702b61 100644 --- a/cmd/restic/cmd_check.go +++ b/cmd/restic/cmd_check.go @@ -342,7 +342,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args p.Done() if len(salvagePacks) > 0 { - Warnf("\nThe repository contains pack files with damaged blobs. These blobs must be removed to repair the repository. This can be done using the following commands:\n\n") + Warnf("\nThe repository contains pack files with damaged blobs. These blobs must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n") var strIDs []string for _, id := range salvagePacks { strIDs = append(strIDs, id.String()) From 9e3703ded525b2d6755fa385b8f82b6f785d1392 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 12 Feb 2024 20:39:31 +0100 Subject: [PATCH 144/158] remove changelogs that are already included in restic 0.16.4 --- changelog/unreleased/issue-4529 | 18 ------------------ changelog/unreleased/issue-4677 | 19 ------------------- 2 files changed, 37 deletions(-) delete mode 100644 changelog/unreleased/issue-4529 delete mode 100644 changelog/unreleased/issue-4677 diff --git a/changelog/unreleased/issue-4529 b/changelog/unreleased/issue-4529 deleted file mode 100644 index fed726d2d..000000000 --- a/changelog/unreleased/issue-4529 +++ /dev/null @@ -1,18 +0,0 @@ -Enhancement: Add extra verification of data integrity before upload - -Hardware issues, or a bug in restic or its dependencies, could previously cause -corruption in the files restic created and stored in the repository. Detecting -such corruption previously required explicitly running the `check --read-data` -or `check --read-data-subset` commands. - -To further ensure data integrity, even in the case of hardware issues or -software bugs, restic now performs additional verification of the files about -to be uploaded to the repository. - -These extra checks will increase CPU usage during backups. They can therefore, -if absolutely necessary, be disabled using the `--no-extra-verify` global -option. Please note that this should be combined with more active checking -using the previously mentioned check commands. - -https://github.com/restic/restic/issues/4529 -https://github.com/restic/restic/pull/4681 diff --git a/changelog/unreleased/issue-4677 b/changelog/unreleased/issue-4677 deleted file mode 100644 index 8fa6cf65b..000000000 --- a/changelog/unreleased/issue-4677 +++ /dev/null @@ -1,19 +0,0 @@ -Bugfix: Downgrade zstd library to fix rare data corruption at max. compression - -In restic 0.16.3, backups where the compression level was set to `max` (using -`--compression max`) could in rare and very specific circumstances result in -data corruption due to a bug in the library used for compressing data. Restic -0.16.1 and 0.16.2 were not affected. - -Restic now uses the previous version of the library used to compress data, the -same version used by restic 0.16.2. Please note that the `auto` compression -level (which restic uses by default) was never affected, and even if you used -`max` compression, chances of being affected by this issue are small. - -To check a repository for any corruption, run `restic check --read-data`. This -will download and verify the whole repository and can be used at any time to -completely verify the integrity of a repository. If the `check` command detects -anomalies, follow the suggested steps. - -https://github.com/restic/restic/issues/4677 -https://github.com/restic/restic/pull/4679 From 69304cd74f9853c36fb6d34a07ae1a42aa552ff4 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 10 Feb 2024 20:29:11 +0100 Subject: [PATCH 145/158] check: clarify repair pack usage --- doc/077_troubleshooting.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/doc/077_troubleshooting.rst b/doc/077_troubleshooting.rst index 512b41b0d..f80df29b8 100644 --- a/doc/077_troubleshooting.rst +++ b/doc/077_troubleshooting.rst @@ -77,7 +77,9 @@ somewhere. Please include the check output and additional information that might help locate the problem. If ``check`` detects damaged pack files, it will show instructions on how to repair -them. Please follow the steps from the next section first. +them using the ``repair pack`` command. Use that command instead of the "Repair the +index" section in this guide. + 2. Backup the repository ************************ @@ -106,6 +108,11 @@ whether your issue is already known and solved. Please take a look at the 3. Repair the index ******************* +.. note:: + + If the `check` command tells you to run `restic repair pack`, then use that + command instead. It will repair the damaged pack files and also update the index. + Restic relies on its index to contain correct information about what data is stored in the repository. Thus, the first step to repair a repository is to repair the index: From 0a36d193d84a13dea9911fb7ad9e071f24db40ec Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Mon, 12 Feb 2024 20:38:20 +0100 Subject: [PATCH 146/158] add changelog for enhanced repair packs --- changelog/unreleased/pull-4644 | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 changelog/unreleased/pull-4644 diff --git a/changelog/unreleased/pull-4644 b/changelog/unreleased/pull-4644 new file mode 100644 index 000000000..8000bce7e --- /dev/null +++ b/changelog/unreleased/pull-4644 @@ -0,0 +1,10 @@ +Enhancement: Improve `repair packs` command + +The `repair packs` command has been improved to also be able to process +truncated pack files. The `check --read-data` command will provide instructions +on using the command if necessary to repair a repository. See the guide at +https://restic.readthedocs.io/en/stable/077_troubleshooting.html for further +instructions. + +https://github.com/restic/restic/pull/4644 +https://github.com/restic/restic/pull/4655 From 18b0bbbf420131f8d6194fbbe89217a100934ff4 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 17 Feb 2024 19:37:32 +0100 Subject: [PATCH 147/158] repository: use fmt.Errorf in StreamPacks --- internal/repository/repository.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 9a4494c4d..87eb01367 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -1079,7 +1079,7 @@ func (b *PackBlobIterator) Next() (PackBlobValue, error) { skipBytes := int(entry.Offset - b.currentOffset) if skipBytes < 0 { - return PackBlobValue{}, errors.Errorf("overlapping blobs in pack %v", b.packID) + return PackBlobValue{}, fmt.Errorf("overlapping blobs in pack %v", b.packID) } _, err := b.rd.Discard(skipBytes) @@ -1099,18 +1099,18 @@ func (b *PackBlobIterator) Next() (PackBlobValue, error) { n, err := io.ReadFull(b.rd, b.buf) if err != nil { debug.Log(" read error %v", err) - return PackBlobValue{}, errors.Wrap(err, "ReadFull") + return PackBlobValue{}, fmt.Errorf("readFull: %w", err) } if n != len(b.buf) { - return PackBlobValue{}, errors.Errorf("read blob %v from %v: not enough bytes read, want %v, got %v", + return PackBlobValue{}, fmt.Errorf("read blob %v from %v: not enough bytes read, want %v, got %v", h, b.packID.Str(), len(b.buf), n) } b.currentOffset = entry.Offset + entry.Length if int(entry.Length) <= b.key.NonceSize() { debug.Log("%v", b.blobs) - return PackBlobValue{}, errors.Errorf("invalid blob length %v", entry) + return PackBlobValue{}, fmt.Errorf("invalid blob length %v", entry) } // decryption errors are likely permanent, give the caller a chance to skip them @@ -1130,7 +1130,7 @@ func (b *PackBlobIterator) Next() (PackBlobValue, error) { if !id.Equal(entry.ID) { debug.Log("read blob %v/%v from %v: wrong data returned, hash is %v", h.Type, h.ID, b.packID.Str(), id) - err = errors.Errorf("read blob %v from %v: wrong data returned, hash is %v", + err = fmt.Errorf("read blob %v from %v: wrong data returned, hash is %v", h, b.packID.Str(), id) } } From 4c3218ef9f35e6a8336a096e45d5df7b57244bc4 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 17 Feb 2024 19:38:01 +0100 Subject: [PATCH 148/158] repository: include packID in StreamPack for decrypt/decompress errors --- internal/repository/repository.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index 87eb01367..8e34c7125 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -1116,13 +1116,16 @@ func (b *PackBlobIterator) Next() (PackBlobValue, error) { // decryption errors are likely permanent, give the caller a chance to skip them nonce, ciphertext := b.buf[:b.key.NonceSize()], b.buf[b.key.NonceSize():] plaintext, err := b.key.Open(ciphertext[:0], nonce, ciphertext, nil) + if err != nil { + err = fmt.Errorf("decrypting blob %v from %v failed: %w", h, b.packID.Str(), err) + } if err == nil && entry.IsCompressed() { // DecodeAll will allocate a slice if it is not large enough since it // knows the decompressed size (because we're using EncodeAll) b.decode, err = b.dec.DecodeAll(plaintext, b.decode[:0]) plaintext = b.decode if err != nil { - err = errors.Errorf("decompressing blob %v failed: %v", h, err) + err = fmt.Errorf("decompressing blob %v from %v failed: %w", h, b.packID.Str(), err) } } if err == nil { From 30e979d2527817eead2cd9786023aa3e247f9186 Mon Sep 17 00:00:00 2001 From: Brian Harring Date: Mon, 19 Feb 2024 11:09:25 +0100 Subject: [PATCH 149/158] Catch SIGTERM, run cleanup The previous code only ran cleanup (lock release for example) on SIGINT. For anyone running restic in a container, the signal is going to be SIGTERM which means containerized execution would leave locks behind. While this could be addressed via interposing dumb-init to translate the signal, a `kill` invocation is going to default to SIGTERM, so the same problem exists for non container users. Signed-off-by: Brian Harring --- cmd/restic/cleanup.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/restic/cleanup.go b/cmd/restic/cleanup.go index 75933fe96..d1c8ecc1a 100644 --- a/cmd/restic/cleanup.go +++ b/cmd/restic/cleanup.go @@ -19,7 +19,7 @@ var cleanupHandlers struct { func init() { cleanupHandlers.ch = make(chan os.Signal, 1) go CleanupHandler(cleanupHandlers.ch) - signal.Notify(cleanupHandlers.ch, syscall.SIGINT) + signal.Notify(cleanupHandlers.ch, syscall.SIGINT, syscall.SIGTERM) } // AddCleanupHandler adds the function f to the list of cleanup handlers so @@ -70,7 +70,7 @@ func CleanupHandler(c <-chan os.Signal) { code := 0 - if s == syscall.SIGINT { + if s == syscall.SIGINT || s == syscall.SIGTERM { code = 130 } else { code = 1 From b41107dcaf214743b0bc6edc445a6d1882a3283c Mon Sep 17 00:00:00 2001 From: Brian Harring Date: Mon, 19 Feb 2024 11:31:48 +0100 Subject: [PATCH 150/158] Add changelog for SIGTERM bugfix. Signed-off-by: Brian Harring --- changelog/unreleased/pull-4703 | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 changelog/unreleased/pull-4703 diff --git a/changelog/unreleased/pull-4703 b/changelog/unreleased/pull-4703 new file mode 100644 index 000000000..4df3385a0 --- /dev/null +++ b/changelog/unreleased/pull-4703 @@ -0,0 +1,9 @@ +Bugfix: Shutdown cleanly when SIGTERM is received + +Prior, if restic received SIGTERM it'd just immediately terminate skipping +cleanup- resulting in potential issues like stale locks being left behind. + +This primarily effected containerized restic invocations- they use SIGTERM- +but this could be triggered via a simple `killall restic` in addition. + +https://github.com/restic/restic/pull/4703 From 0a65a0f94fa6d22a32e227024f34c23126b754e3 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Thu, 22 Feb 2024 22:00:42 +0100 Subject: [PATCH 151/158] update comment --- cmd/restic/cleanup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/restic/cleanup.go b/cmd/restic/cleanup.go index d1c8ecc1a..5a6cf79e1 100644 --- a/cmd/restic/cleanup.go +++ b/cmd/restic/cleanup.go @@ -56,7 +56,7 @@ func RunCleanupHandlers(code int) int { return code } -// CleanupHandler handles the SIGINT signals. +// CleanupHandler handles the SIGINT and SIGTERM signals. func CleanupHandler(c <-chan os.Signal) { for s := range c { debug.Log("signal %v received, cleaning up", s) From 94de87d4b7d6b12ffbe3726954ed4b02d4360a3a Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia <99904+aneesh-n@users.noreply.github.com> Date: Thu, 22 Feb 2024 16:57:00 -0700 Subject: [PATCH 152/158] Add CombineErrors helper function --- internal/errors/errors.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/internal/errors/errors.go b/internal/errors/errors.go index 0327ea0da..3c669f861 100644 --- a/internal/errors/errors.go +++ b/internal/errors/errors.go @@ -2,6 +2,7 @@ package errors import ( stderrors "errors" + "fmt" "github.com/pkg/errors" ) @@ -22,12 +23,42 @@ var Wrap = errors.Wrap // nil, Wrapf returns nil. var Wrapf = errors.Wrapf +// WithStack annotates err with a stack trace at the point WithStack was called. +// If err is nil, WithStack returns nil. var WithStack = errors.WithStack // Go 1.13-style error handling. +// As finds the first error in err's tree that matches target, and if one is found, +// sets target to that error value and returns true. Otherwise, it returns false. func As(err error, tgt interface{}) bool { return stderrors.As(err, tgt) } +// Is reports whether any error in err's tree matches target. func Is(x, y error) bool { return stderrors.Is(x, y) } +// Unwrap returns the result of calling the Unwrap method on err, if err's type contains +// an Unwrap method returning error. Otherwise, Unwrap returns nil. +// +// Unwrap only calls a method of the form "Unwrap() error". In particular Unwrap does not +// unwrap errors returned by [Join]. func Unwrap(err error) error { return stderrors.Unwrap(err) } + +// CombineErrors combines multiple errors into a single error. +func CombineErrors(errors ...error) error { + var combinedErrorMsg string + + for _, err := range errors { + if err != nil { + if combinedErrorMsg != "" { + combinedErrorMsg += "; " // Separate error messages with a delimiter + } + combinedErrorMsg += err.Error() + } + } + + if combinedErrorMsg == "" { + return nil // No errors, return nil + } + + return fmt.Errorf("multiple errors occurred: [%s]", combinedErrorMsg) +} From 62a8a599f1f4f36c55e17106a8e40bd956430058 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia <99904+aneesh-n@users.noreply.github.com> Date: Thu, 22 Feb 2024 16:58:12 -0700 Subject: [PATCH 153/158] Add optional messages for Equals helper --- internal/test/helpers.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/internal/test/helpers.go b/internal/test/helpers.go index 65e3e36ec..242da6079 100644 --- a/internal/test/helpers.go +++ b/internal/test/helpers.go @@ -3,6 +3,7 @@ package test import ( "compress/bzip2" "compress/gzip" + "fmt" "io" "os" "os/exec" @@ -47,10 +48,22 @@ func OKs(tb testing.TB, errs []error) { } // Equals fails the test if exp is not equal to act. -func Equals(tb testing.TB, exp, act interface{}) { +// msg is optional message to be printed, first param being format string and rest being arguments. +func Equals(tb testing.TB, exp, act interface{}, msgs ...string) { tb.Helper() if !reflect.DeepEqual(exp, act) { - tb.Fatalf("\033[31m\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", exp, act) + var msgString string + length := len(msgs) + if length == 1 { + msgString = msgs[0] + } else if length > 1 { + args := make([]interface{}, length-1) + for i, msg := range msgs[1:] { + args[i] = msg + } + msgString = fmt.Sprintf(msgs[0], args...) + } + tb.Fatalf("\033[31m\n\n\t"+msgString+"\n\n\texp: %#v\n\n\tgot: %#v\033[39m\n\n", exp, act) } } From 0962917974952531347f82dffc0d6e82b1edf5f5 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia <99904+aneesh-n@users.noreply.github.com> Date: Thu, 22 Feb 2024 17:31:20 -0700 Subject: [PATCH 154/158] Support windows metadata using generic attribs Add new generic_attributes attribute in Node. Use the generic attributes to add support for creation time and file attributes like hidden, readonly, encrypted in windows. Handle permission errors for readonly files in windows. Handle backup and restore of encrypted attributes using windows system calls. --- cmd/restic/cmd_find.go | 1 + internal/fs/file.go | 14 ++ internal/fs/file_windows.go | 26 ++++ internal/restic/node.go | 247 +++++++++++++++++++++++++++++-- internal/restic/node_aix.go | 17 ++- internal/restic/node_netbsd.go | 24 ++- internal/restic/node_openbsd.go | 24 ++- internal/restic/node_test.go | 35 +++-- internal/restic/node_windows.go | 224 +++++++++++++++++++++++++++- internal/restic/node_xattr.go | 11 ++ internal/restorer/fileswriter.go | 28 ++-- 11 files changed, 591 insertions(+), 60 deletions(-) diff --git a/cmd/restic/cmd_find.go b/cmd/restic/cmd_find.go index 04e6ae3dd..7ea7c425a 100644 --- a/cmd/restic/cmd_find.go +++ b/cmd/restic/cmd_find.go @@ -126,6 +126,7 @@ func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) { // Make the following attributes disappear Name byte `json:"name,omitempty"` ExtendedAttributes byte `json:"extended_attributes,omitempty"` + GenericAttributes byte `json:"generic_attributes,omitempty"` Device byte `json:"device,omitempty"` Content byte `json:"content,omitempty"` Subtree byte `json:"subtree,omitempty"` diff --git a/internal/fs/file.go b/internal/fs/file.go index f35901c06..4a236ea09 100644 --- a/internal/fs/file.go +++ b/internal/fs/file.go @@ -124,3 +124,17 @@ func RemoveIfExists(filename string) error { func Chtimes(name string, atime time.Time, mtime time.Time) error { return os.Chtimes(fixpath(name), atime, mtime) } + +// IsAccessDenied checks if the error is due to permission error. +func IsAccessDenied(err error) bool { + return os.IsPermission(err) +} + +// ResetPermissions resets the permissions of the file at the specified path +func ResetPermissions(path string) error { + // Set the default file permissions + if err := os.Chmod(path, 0600); err != nil { + return err + } + return nil +} diff --git a/internal/fs/file_windows.go b/internal/fs/file_windows.go index d19a744e1..2f0969804 100644 --- a/internal/fs/file_windows.go +++ b/internal/fs/file_windows.go @@ -77,3 +77,29 @@ func TempFile(dir, prefix string) (f *os.File, err error) { func Chmod(name string, mode os.FileMode) error { return os.Chmod(fixpath(name), mode) } + +// ClearSystem removes the system attribute from the file. +func ClearSystem(path string) error { + return ClearAttribute(path, windows.FILE_ATTRIBUTE_SYSTEM) +} + +// ClearAttribute removes the specified attribute from the file. +func ClearAttribute(path string, attribute uint32) error { + ptr, err := windows.UTF16PtrFromString(path) + if err != nil { + return err + } + fileAttributes, err := windows.GetFileAttributes(ptr) + if err != nil { + return err + } + if fileAttributes&attribute != 0 { + // Clear the attribute + fileAttributes &= ^uint32(attribute) + err = windows.SetFileAttributes(ptr, fileAttributes) + if err != nil { + return err + } + } + return nil +} diff --git a/internal/restic/node.go b/internal/restic/node.go index 1d5bb51af..44ca52b0c 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -6,7 +6,9 @@ import ( "fmt" "os" "os/user" + "reflect" "strconv" + "strings" "sync" "syscall" "time" @@ -20,12 +22,53 @@ import ( "github.com/restic/restic/internal/fs" ) -// ExtendedAttribute is a tuple storing the xattr name and value. +// ExtendedAttribute is a tuple storing the xattr name and value for various filesystems. type ExtendedAttribute struct { Name string `json:"name"` Value []byte `json:"value"` } +// GenericAttributeType can be used for OS specific functionalities by defining specific types +// in node.go to be used by the specific node_xx files. +// OS specific attribute types should follow the convention Attributes. +// GenericAttributeTypes should follow the convention . +// The attributes in OS specific attribute types must be pointers as we want to distinguish nil values +// and not create GenericAttributes for them. +type GenericAttributeType string + +// OSType is the type created to represent each specific OS +type OSType string + +const ( + // When new GenericAttributeType are defined, they must be added in the init function as well. + + // Below are windows specific attributes. + + // TypeCreationTime is the GenericAttributeType used for storing creation time for windows files within the generic attributes map. + TypeCreationTime GenericAttributeType = "windows.creation_time" + // TypeFileAttributes is the GenericAttributeType used for storing file attributes for windows files within the generic attributes map. + TypeFileAttributes GenericAttributeType = "windows.file_attributes" + + // Generic Attributes for other OS types should be defined here. +) + +// init is called when the package is initialized. Any new GenericAttributeTypes being created must be added here as well. +func init() { + storeGenericAttributeType(TypeCreationTime, TypeFileAttributes) +} + +// genericAttributesForOS maintains a map of known genericAttributesForOS to the OSType +var genericAttributesForOS = map[GenericAttributeType]OSType{} + +// storeGenericAttributeType adds and entry in genericAttributesForOS map +func storeGenericAttributeType(attributeTypes ...GenericAttributeType) { + for _, attributeType := range attributeTypes { + // Get the OS attribute type from the GenericAttributeType + osAttributeName := strings.Split(string(attributeType), ".")[0] + genericAttributesForOS[attributeType] = OSType(osAttributeName) + } +} + // Node is a file, directory or other item in a backup. type Node struct { Name string `json:"name"` @@ -47,11 +90,12 @@ type Node struct { // This allows storing arbitrary byte-sequences, which are possible as symlink targets on unix systems, // as LinkTarget without breaking backwards-compatibility. // Must only be set of the linktarget cannot be encoded as valid utf8. - LinkTargetRaw []byte `json:"linktarget_raw,omitempty"` - ExtendedAttributes []ExtendedAttribute `json:"extended_attributes,omitempty"` - Device uint64 `json:"device,omitempty"` // in case of Type == "dev", stat.st_rdev - Content IDs `json:"content"` - Subtree *ID `json:"subtree,omitempty"` + LinkTargetRaw []byte `json:"linktarget_raw,omitempty"` + ExtendedAttributes []ExtendedAttribute `json:"extended_attributes,omitempty"` + GenericAttributes map[GenericAttributeType]json.RawMessage `json:"generic_attributes,omitempty"` + Device uint64 `json:"device,omitempty"` // in case of Type == "dev", stat.st_rdev + Content IDs `json:"content"` + Subtree *ID `json:"subtree,omitempty"` Error string `json:"error,omitempty"` @@ -203,14 +247,6 @@ func (node Node) restoreMetadata(path string) error { } } - if node.Type != "symlink" { - if err := fs.Chmod(path, node.Mode); err != nil { - if firsterr != nil { - firsterr = errors.WithStack(err) - } - } - } - if err := node.RestoreTimestamps(path); err != nil { debug.Log("error restoring timestamps for dir %v: %v", path, err) if firsterr != nil { @@ -225,6 +261,24 @@ func (node Node) restoreMetadata(path string) error { } } + if err := node.restoreGenericAttributes(path); err != nil { + debug.Log("error restoring generic attributes for %v: %v", path, err) + if firsterr != nil { + firsterr = err + } + } + + // Moving RestoreTimestamps and restoreExtendedAttributes calls above as for readonly files in windows + // calling Chmod below will no longer allow any modifications to be made on the file and the + // calls above would fail. + if node.Type != "symlink" { + if err := fs.Chmod(path, node.Mode); err != nil { + if firsterr != nil { + firsterr = errors.WithStack(err) + } + } + } + return firsterr } @@ -438,6 +492,9 @@ func (node Node) Equals(other Node) bool { if !node.sameExtendedAttributes(other) { return false } + if !node.sameGenericAttributes(other) { + return false + } if node.Subtree != nil { if other.Subtree == nil { return false @@ -480,8 +537,13 @@ func (node Node) sameContent(other Node) bool { } func (node Node) sameExtendedAttributes(other Node) bool { - if len(node.ExtendedAttributes) != len(other.ExtendedAttributes) { + ln := len(node.ExtendedAttributes) + lo := len(other.ExtendedAttributes) + if ln != lo { return false + } else if ln == 0 { + // This means lo is also of length 0 + return true } // build a set of all attributes that node has @@ -525,6 +587,33 @@ func (node Node) sameExtendedAttributes(other Node) bool { return true } +func (node Node) sameGenericAttributes(other Node) bool { + return deepEqual(node.GenericAttributes, other.GenericAttributes) +} + +func deepEqual(map1, map2 map[GenericAttributeType]json.RawMessage) bool { + // Check if the maps have the same number of keys + if len(map1) != len(map2) { + return false + } + + // Iterate over each key-value pair in map1 + for key, value1 := range map1 { + // Check if the key exists in map2 + value2, ok := map2[key] + if !ok { + return false + } + + // Check if the JSON.RawMessage values are equal byte by byte + if !bytes.Equal(value1, value2) { + return false + } + } + + return true +} + func (node *Node) fillUser(stat *statT) { uid, gid := stat.uid(), stat.gid() node.UID, node.GID = uid, gid @@ -627,7 +716,17 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error { return errors.Errorf("unsupported file type %q", node.Type) } - return node.fillExtendedAttributes(path) + allowExtended, err := node.fillGenericAttributes(path, fi, stat) + if allowExtended { + // Skip processing ExtendedAttributes if allowExtended is false. + errEx := node.fillExtendedAttributes(path) + if err == nil { + err = errEx + } else { + debug.Log("Error filling extended attributes for %v at %v : %v", node.Name, path, errEx) + } + } + return err } func (node *Node) fillExtendedAttributes(path string) error { @@ -665,3 +764,119 @@ func (node *Node) fillTimes(stat *statT) { node.ChangeTime = time.Unix(ctim.Unix()) node.AccessTime = time.Unix(atim.Unix()) } + +// HandleUnknownGenericAttributesFound is used for handling and distinguing between scenarios related to future versions and cross-OS repositories +func HandleUnknownGenericAttributesFound(unknownAttribs []GenericAttributeType) { + for _, unknownAttrib := range unknownAttribs { + handleUnknownGenericAttributeFound(unknownAttrib) + } +} + +// handleUnknownGenericAttributeFound is used for handling and distinguing between scenarios related to future versions and cross-OS repositories +func handleUnknownGenericAttributeFound(genericAttributeType GenericAttributeType) { + if checkGenericAttributeNameNotHandledAndPut(genericAttributeType) { + // Print the unique error only once for a given execution + os, exists := genericAttributesForOS[genericAttributeType] + + if exists { + // If genericAttributesForOS contains an entry but we still got here, it means the specific node_xx.go for the current OS did not handle it and the repository may have been originally created on a different OS. + // The fact that node.go knows about the attribute, means it is not a new attribute. This may be a common situation if a repo is used across OSs. + debug.Log("Ignoring a generic attribute found in the repository: %s which may not be compatible with your OS. Compatible OS: %s", genericAttributeType, os) + } else { + // If genericAttributesForOS in node.go does not know about this attribute, then the repository may have been created by a newer version which has a newer GenericAttributeType. + debug.Log("Found an unrecognized generic attribute in the repository: %s. You may need to upgrade to latest version of restic.", genericAttributeType) + } + } +} + +// handleAllUnknownGenericAttributesFound performs validations for all generic attributes in the node. +// This is not used on windows currently because windows has handling for generic attributes. +// nolint:unused +func (node Node) handleAllUnknownGenericAttributesFound() error { + for name := range node.GenericAttributes { + handleUnknownGenericAttributeFound(name) + } + return nil +} + +var unknownGenericAttributesHandlingHistory sync.Map + +// checkGenericAttributeNameNotHandledAndPut checks if the GenericAttributeType name entry +// already exists and puts it in the map if not. +func checkGenericAttributeNameNotHandledAndPut(value GenericAttributeType) bool { + // If Key doesn't exist, put the value and return true because it is not already handled + _, exists := unknownGenericAttributesHandlingHistory.LoadOrStore(value, "") + // Key exists, then it is already handled so return false + return !exists +} + +// The functions below are common helper functions which can be used for generic attributes support +// across different OS. + +// genericAttributesToOSAttrs gets the os specific attribute from the generic attribute using reflection +// nolint:unused +func genericAttributesToOSAttrs(attrs map[GenericAttributeType]json.RawMessage, attributeType reflect.Type, attributeValuePtr *reflect.Value, keyPrefix string) (unknownAttribs []GenericAttributeType, err error) { + attributeValue := *attributeValuePtr + + for key, rawMsg := range attrs { + found := false + for i := 0; i < attributeType.NumField(); i++ { + if getFQKeyByIndex(attributeType, i, keyPrefix) == key { + found = true + fieldValue := attributeValue.Field(i) + // For directly supported types, use json.Unmarshal directly + if err := json.Unmarshal(rawMsg, fieldValue.Addr().Interface()); err != nil { + return unknownAttribs, errors.Wrap(err, "Unmarshal") + } + break + } + } + if !found { + unknownAttribs = append(unknownAttribs, key) + } + } + return unknownAttribs, nil +} + +// getFQKey gets the fully qualified key for the field +// nolint:unused +func getFQKey(field reflect.StructField, keyPrefix string) GenericAttributeType { + return GenericAttributeType(fmt.Sprintf("%s.%s", keyPrefix, field.Tag.Get("generic"))) +} + +// getFQKeyByIndex gets the fully qualified key for the field index +// nolint:unused +func getFQKeyByIndex(attributeType reflect.Type, index int, keyPrefix string) GenericAttributeType { + return getFQKey(attributeType.Field(index), keyPrefix) +} + +// osAttrsToGenericAttributes gets the generic attribute from the os specific attribute using reflection +// nolint:unused +func osAttrsToGenericAttributes(attributeType reflect.Type, attributeValuePtr *reflect.Value, keyPrefix string) (attrs map[GenericAttributeType]json.RawMessage, err error) { + attributeValue := *attributeValuePtr + attrs = make(map[GenericAttributeType]json.RawMessage) + + // Iterate over the fields of the struct + for i := 0; i < attributeType.NumField(); i++ { + field := attributeType.Field(i) + + // Get the field value using reflection + fieldValue := attributeValue.FieldByName(field.Name) + + // Check if the field is nil + if fieldValue.IsNil() { + // If it's nil, skip this field + continue + } + + // Marshal the field value into a json.RawMessage + var fieldBytes []byte + if fieldBytes, err = json.Marshal(fieldValue.Interface()); err != nil { + return attrs, errors.Wrap(err, "Marshal") + } + + // Insert the field into the map + attrs[getFQKey(field, keyPrefix)] = json.RawMessage(fieldBytes) + } + return attrs, nil +} diff --git a/internal/restic/node_aix.go b/internal/restic/node_aix.go index 572e33a65..4d8c248de 100644 --- a/internal/restic/node_aix.go +++ b/internal/restic/node_aix.go @@ -3,9 +3,12 @@ package restic -import "syscall" +import ( + "os" + "syscall" +) -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { return nil } @@ -34,3 +37,13 @@ func Listxattr(path string) ([]string, error) { func Setxattr(path, name string, data []byte) error { return nil } + +// restoreGenericAttributes is no-op on AIX. +func (node *Node) restoreGenericAttributes(_ string) error { + return node.handleAllUnknownGenericAttributesFound() +} + +// fillGenericAttributes is a no-op on AIX. +func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { + return true, nil +} diff --git a/internal/restic/node_netbsd.go b/internal/restic/node_netbsd.go index 0eade2f37..be4afa3ae 100644 --- a/internal/restic/node_netbsd.go +++ b/internal/restic/node_netbsd.go @@ -1,8 +1,11 @@ package restic -import "syscall" +import ( + "os" + "syscall" +) -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { return nil } @@ -10,18 +13,27 @@ func (s statT) atim() syscall.Timespec { return s.Atimespec } func (s statT) mtim() syscall.Timespec { return s.Mtimespec } func (s statT) ctim() syscall.Timespec { return s.Ctimespec } -// Getxattr retrieves extended attribute data associated with path. +// Getxattr is a no-op on netbsd. func Getxattr(path, name string) ([]byte, error) { return nil, nil } -// Listxattr retrieves a list of names of extended attributes associated with the -// given path in the file system. +// Listxattr is a no-op on netbsd. func Listxattr(path string) ([]string, error) { return nil, nil } -// Setxattr associates name and data together as an attribute of path. +// Setxattr is a no-op on netbsd. func Setxattr(path, name string, data []byte) error { return nil } + +// restoreGenericAttributes is no-op on netbsd. +func (node *Node) restoreGenericAttributes(_ string) error { + return node.handleAllUnknownGenericAttributesFound() +} + +// fillGenericAttributes is a no-op on netbsd. +func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { + return true, nil +} diff --git a/internal/restic/node_openbsd.go b/internal/restic/node_openbsd.go index a4ccc7211..bfff8f8aa 100644 --- a/internal/restic/node_openbsd.go +++ b/internal/restic/node_openbsd.go @@ -1,8 +1,11 @@ package restic -import "syscall" +import ( + "os" + "syscall" +) -func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { +func (node Node) restoreSymlinkTimestamps(_ string, _ [2]syscall.Timespec) error { return nil } @@ -10,18 +13,27 @@ func (s statT) atim() syscall.Timespec { return s.Atim } func (s statT) mtim() syscall.Timespec { return s.Mtim } func (s statT) ctim() syscall.Timespec { return s.Ctim } -// Getxattr retrieves extended attribute data associated with path. +// Getxattr is a no-op on openbsd. func Getxattr(path, name string) ([]byte, error) { return nil, nil } -// Listxattr retrieves a list of names of extended attributes associated with the -// given path in the file system. +// Listxattr is a no-op on openbsd. func Listxattr(path string) ([]string, error) { return nil, nil } -// Setxattr associates name and data together as an attribute of path. +// Setxattr is a no-op on openbsd. func Setxattr(path, name string, data []byte) error { return nil } + +// restoreGenericAttributes is no-op on openbsd. +func (node *Node) restoreGenericAttributes(_ string) error { + return node.handleAllUnknownGenericAttributesFound() +} + +// fillGenericAttributes is a no-op on openbsd. +func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { + return true, nil +} diff --git a/internal/restic/node_test.go b/internal/restic/node_test.go index aae010421..c2c7306b7 100644 --- a/internal/restic/node_test.go +++ b/internal/restic/node_test.go @@ -1,4 +1,4 @@ -package restic_test +package restic import ( "context" @@ -11,7 +11,6 @@ import ( "testing" "time" - "github.com/restic/restic/internal/restic" "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test" ) @@ -32,7 +31,7 @@ func BenchmarkNodeFillUser(t *testing.B) { t.ResetTimer() for i := 0; i < t.N; i++ { - _, err := restic.NodeFromFileInfo(path, fi) + _, err := NodeFromFileInfo(path, fi) rtest.OK(t, err) } @@ -56,7 +55,7 @@ func BenchmarkNodeFromFileInfo(t *testing.B) { t.ResetTimer() for i := 0; i < t.N; i++ { - _, err := restic.NodeFromFileInfo(path, fi) + _, err := NodeFromFileInfo(path, fi) if err != nil { t.Fatal(err) } @@ -75,11 +74,11 @@ func parseTime(s string) time.Time { return t.Local() } -var nodeTests = []restic.Node{ +var nodeTests = []Node{ { Name: "testFile", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0604, @@ -90,7 +89,7 @@ var nodeTests = []restic.Node{ { Name: "testSuidFile", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0755 | os.ModeSetuid, @@ -101,7 +100,7 @@ var nodeTests = []restic.Node{ { Name: "testSuidFile2", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0755 | os.ModeSetgid, @@ -112,7 +111,7 @@ var nodeTests = []restic.Node{ { Name: "testSticky", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0755 | os.ModeSticky, @@ -148,7 +147,7 @@ var nodeTests = []restic.Node{ { Name: "testFile", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0604, @@ -170,14 +169,14 @@ var nodeTests = []restic.Node{ { Name: "testXattrFile", Type: "file", - Content: restic.IDs{}, + Content: IDs{}, UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), Mode: 0604, ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []restic.ExtendedAttribute{ + ExtendedAttributes: []ExtendedAttribute{ {"user.foo", []byte("bar")}, }, }, @@ -191,7 +190,7 @@ var nodeTests = []restic.Node{ ModTime: parseTime("2005-05-14 21:07:03.111"), AccessTime: parseTime("2005-05-14 21:07:04.222"), ChangeTime: parseTime("2005-05-14 21:07:05.333"), - ExtendedAttributes: []restic.ExtendedAttribute{ + ExtendedAttributes: []ExtendedAttribute{ {"user.foo", []byte("bar")}, }, }, @@ -228,7 +227,7 @@ func TestNodeRestoreAt(t *testing.T) { fi, err := os.Lstat(nodePath) rtest.OK(t, err) - n2, err := restic.NodeFromFileInfo(nodePath, fi) + n2, err := NodeFromFileInfo(nodePath, fi) rtest.OK(t, err) rtest.Assert(t, test.Name == n2.Name, @@ -330,7 +329,7 @@ func TestFixTime(t *testing.T) { for _, test := range tests { t.Run("", func(t *testing.T) { - res := restic.FixTime(test.src) + res := FixTime(test.src) if !res.Equal(test.want) { t.Fatalf("wrong result for %v, want:\n %v\ngot:\n %v", test.src, test.want, res) } @@ -343,12 +342,12 @@ func TestSymlinkSerialization(t *testing.T) { "válîd \t Üñi¢òde \n śẗŕinǵ", string([]byte{0, 1, 2, 0xfa, 0xfb, 0xfc}), } { - n := restic.Node{ + n := Node{ LinkTarget: link, } ser, err := json.Marshal(n) test.OK(t, err) - var n2 restic.Node + var n2 Node err = json.Unmarshal(ser, &n2) test.OK(t, err) fmt.Println(string(ser)) @@ -365,7 +364,7 @@ func TestSymlinkSerializationFormat(t *testing.T) { {`{"linktarget":"test"}`, "test"}, {`{"linktarget":"\u0000\u0001\u0002\ufffd\ufffd\ufffd","linktarget_raw":"AAEC+vv8"}`, string([]byte{0, 1, 2, 0xfa, 0xfb, 0xfc})}, } { - var n2 restic.Node + var n2 Node err := json.Unmarshal([]byte(d.ser), &n2) test.OK(t, err) test.Equals(t, d.linkTarget, n2.LinkTarget) diff --git a/internal/restic/node_windows.go b/internal/restic/node_windows.go index fc6439b40..a2b8c75e5 100644 --- a/internal/restic/node_windows.go +++ b/internal/restic/node_windows.go @@ -1,21 +1,47 @@ package restic import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "runtime" + "strings" "syscall" + "unsafe" + "github.com/restic/restic/internal/debug" "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/fs" + "golang.org/x/sys/windows" +) + +// WindowsAttributes are the genericAttributes for Windows OS +type WindowsAttributes struct { + // CreationTime is used for storing creation time for windows files. + CreationTime *syscall.Filetime `generic:"creation_time"` + // FileAttributes is used for storing file attributes for windows files. + FileAttributes *uint32 `generic:"file_attributes"` +} + +var ( + modAdvapi32 = syscall.NewLazyDLL("advapi32.dll") + procEncryptFile = modAdvapi32.NewProc("EncryptFileW") + procDecryptFile = modAdvapi32.NewProc("DecryptFileW") ) // mknod is not supported on Windows. -func mknod(path string, mode uint32, dev uint64) (err error) { +func mknod(_ string, mode uint32, dev uint64) (err error) { return errors.New("device nodes cannot be created on windows") } // Windows doesn't need lchown -func lchown(path string, uid int, gid int) (err error) { +func lchown(_ string, uid int, gid int) (err error) { return nil } +// restoreSymlinkTimestamps restores timestamps for symlinks func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespec) error { // tweaked version of UtimesNano from go/src/syscall/syscall_windows.go pathp, e := syscall.UTF16PtrFromString(path) @@ -28,7 +54,14 @@ func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespe if e != nil { return e } - defer syscall.Close(h) + + defer func() { + err := syscall.Close(h) + if err != nil { + debug.Log("Error closing file handle for %s: %v\n", path, err) + } + }() + a := syscall.NsecToFiletime(syscall.TimespecToNsec(utimes[0])) w := syscall.NsecToFiletime(syscall.TimespecToNsec(utimes[1])) return syscall.SetFileTime(h, nil, &a, &w) @@ -83,3 +116,188 @@ func (s statT) ctim() syscall.Timespec { // Windows does not have the concept of a "change time" in the sense Unix uses it, so we're using the LastWriteTime here. return syscall.NsecToTimespec(s.LastWriteTime.Nanoseconds()) } + +// restoreGenericAttributes restores generic attributes for Windows +func (node Node) restoreGenericAttributes(path string) (err error) { + if len(node.GenericAttributes) == 0 { + return nil + } + var errs []error + windowsAttributes, unknownAttribs, err := genericAttributesToWindowsAttrs(node.GenericAttributes) + if err != nil { + return fmt.Errorf("error parsing generic attribute for: %s : %v", path, err) + } + if windowsAttributes.CreationTime != nil { + if err := restoreCreationTime(path, windowsAttributes.CreationTime); err != nil { + errs = append(errs, fmt.Errorf("error restoring creation time for: %s : %v", path, err)) + } + } + if windowsAttributes.FileAttributes != nil { + if err := restoreFileAttributes(path, windowsAttributes.FileAttributes); err != nil { + errs = append(errs, fmt.Errorf("error restoring file attributes for: %s : %v", path, err)) + } + } + + HandleUnknownGenericAttributesFound(unknownAttribs) + return errors.CombineErrors(errs...) +} + +// genericAttributesToWindowsAttrs converts the generic attributes map to a WindowsAttributes and also returns a string of unkown attributes that it could not convert. +func genericAttributesToWindowsAttrs(attrs map[GenericAttributeType]json.RawMessage) (windowsAttributes WindowsAttributes, unknownAttribs []GenericAttributeType, err error) { + waValue := reflect.ValueOf(&windowsAttributes).Elem() + unknownAttribs, err = genericAttributesToOSAttrs(attrs, reflect.TypeOf(windowsAttributes), &waValue, "windows") + return windowsAttributes, unknownAttribs, err +} + +// restoreCreationTime gets the creation time from the data and sets it to the file/folder at +// the specified path. +func restoreCreationTime(path string, creationTime *syscall.Filetime) (err error) { + pathPointer, err := syscall.UTF16PtrFromString(path) + if err != nil { + return err + } + handle, err := syscall.CreateFile(pathPointer, + syscall.FILE_WRITE_ATTRIBUTES, syscall.FILE_SHARE_WRITE, nil, + syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS, 0) + if err != nil { + return err + } + defer func() { + if err := syscall.Close(handle); err != nil { + debug.Log("Error closing file handle for %s: %v\n", path, err) + } + }() + return syscall.SetFileTime(handle, creationTime, nil, nil) +} + +// restoreFileAttributes gets the File Attributes from the data and sets them to the file/folder +// at the specified path. +func restoreFileAttributes(path string, fileAttributes *uint32) (err error) { + pathPointer, err := syscall.UTF16PtrFromString(path) + if err != nil { + return err + } + err = fixEncryptionAttribute(path, fileAttributes, pathPointer) + if err != nil { + debug.Log("Could not change encryption attribute for path: %s: %v", path, err) + } + return syscall.SetFileAttributes(pathPointer, *fileAttributes) +} + +// fixEncryptionAttribute checks if a file needs to be marked encrypted and is not already encrypted, it sets +// the FILE_ATTRIBUTE_ENCRYPTED. Conversely, if the file needs to be marked unencrypted and it is already +// marked encrypted, it removes the FILE_ATTRIBUTE_ENCRYPTED. +func fixEncryptionAttribute(path string, attrs *uint32, pathPointer *uint16) (err error) { + if *attrs&windows.FILE_ATTRIBUTE_ENCRYPTED != 0 { + // File should be encrypted. + err = encryptFile(pathPointer) + if err != nil { + if fs.IsAccessDenied(err) { + // If existing file already has readonly or system flag, encrypt file call fails. + // We have already cleared readonly flag, clearing system flag if needed. + // The readonly and system flags will be set again at the end of this func if they are needed. + err = fs.ClearSystem(path) + if err != nil { + return fmt.Errorf("failed to encrypt file: failed to clear system flag: %s : %v", path, err) + } + err = encryptFile(pathPointer) + if err != nil { + return fmt.Errorf("failed to encrypt file: %s : %v", path, err) + } + } else { + return fmt.Errorf("failed to encrypt file: %s : %v", path, err) + } + } + } else { + existingAttrs, err := windows.GetFileAttributes(pathPointer) + if err != nil { + return fmt.Errorf("failed to get file attributes for existing file: %s : %v", path, err) + } + if existingAttrs&windows.FILE_ATTRIBUTE_ENCRYPTED != 0 { + // File should not be encrypted, but its already encrypted. Decrypt it. + err = decryptFile(pathPointer) + if err != nil { + if fs.IsAccessDenied(err) { + // If existing file already has readonly or system flag, decrypt file call fails. + // We have already cleared readonly flag, clearing system flag if needed. + // The readonly and system flags will be set again after this func if they are needed. + err = fs.ClearSystem(path) + if err != nil { + return fmt.Errorf("failed to decrypt file: failed to clear system flag: %s : %v", path, err) + } + err = decryptFile(pathPointer) + if err != nil { + return fmt.Errorf("failed to decrypt file: %s : %v", path, err) + } + } else { + return fmt.Errorf("failed to decrypt file: %s : %v", path, err) + } + } + } + } + return err +} + +// encryptFile set the encrypted flag on the file. +func encryptFile(pathPointer *uint16) error { + // Call EncryptFile function + ret, _, err := procEncryptFile.Call(uintptr(unsafe.Pointer(pathPointer))) + if ret == 0 { + return err + } + return nil +} + +// decryptFile removes the encrypted flag from the file. +func decryptFile(pathPointer *uint16) error { + // Call DecryptFile function + ret, _, err := procDecryptFile.Call(uintptr(unsafe.Pointer(pathPointer))) + if ret == 0 { + return err + } + return nil +} + +// fillGenericAttributes fills in the generic attributes for windows like File Attributes, +// Created time etc. +func (node *Node) fillGenericAttributes(path string, fi os.FileInfo, stat *statT) (allowExtended bool, err error) { + if strings.Contains(filepath.Base(path), ":") { + //Do not process for Alternate Data Streams in Windows + // Also do not allow processing of extended attributes for ADS. + return false, nil + } + if !strings.HasSuffix(filepath.Clean(path), `\`) { + // Do not process file attributes and created time for windows directories like + // C:, D: + // Filepath.Clean(path) ends with '\' for Windows root drives only. + + // Add Windows attributes + node.GenericAttributes, err = WindowsAttrsToGenericAttributes(WindowsAttributes{ + CreationTime: getCreationTime(fi, path), + FileAttributes: &stat.FileAttributes, + }) + } + return true, err +} + +// windowsAttrsToGenericAttributes converts the WindowsAttributes to a generic attributes map using reflection +func WindowsAttrsToGenericAttributes(windowsAttributes WindowsAttributes) (attrs map[GenericAttributeType]json.RawMessage, err error) { + // Get the value of the WindowsAttributes + windowsAttributesValue := reflect.ValueOf(windowsAttributes) + return osAttrsToGenericAttributes(reflect.TypeOf(windowsAttributes), &windowsAttributesValue, runtime.GOOS) +} + +// getCreationTime gets the value for the WindowsAttribute CreationTime in a windows specific time format. +// The value is a 64-bit value representing the number of 100-nanosecond intervals since January 1, 1601 (UTC) +// split into two 32-bit parts: the low-order DWORD and the high-order DWORD for efficiency and interoperability. +// The low-order DWORD represents the number of 100-nanosecond intervals elapsed since January 1, 1601, modulo +// 2^32. The high-order DWORD represents the number of times the low-order DWORD has overflowed. +func getCreationTime(fi os.FileInfo, path string) (creationTimeAttribute *syscall.Filetime) { + attrib, success := fi.Sys().(*syscall.Win32FileAttributeData) + if success && attrib != nil { + return &attrib.CreationTime + } else { + debug.Log("Could not get create time for path: %s", path) + return nil + } +} diff --git a/internal/restic/node_xattr.go b/internal/restic/node_xattr.go index ea9eafe94..826b8b74a 100644 --- a/internal/restic/node_xattr.go +++ b/internal/restic/node_xattr.go @@ -4,6 +4,7 @@ package restic import ( + "os" "syscall" "github.com/restic/restic/internal/errors" @@ -47,3 +48,13 @@ func handleXattrErr(err error) error { return errors.WithStack(e) } } + +// restoreGenericAttributes is no-op. +func (node *Node) restoreGenericAttributes(_ string) error { + return node.handleAllUnknownGenericAttributesFound() +} + +// fillGenericAttributes is a no-op. +func (node *Node) fillGenericAttributes(_ string, _ os.FileInfo, _ *statT) (allowExtended bool, err error) { + return true, nil +} diff --git a/internal/restorer/fileswriter.go b/internal/restorer/fileswriter.go index 589aa502a..cbe89c30c 100644 --- a/internal/restorer/fileswriter.go +++ b/internal/restorer/fileswriter.go @@ -50,16 +50,26 @@ func (w *filesWriter) writeToFile(path string, blob []byte, offset int64, create bucket.files[path].users++ return wr, nil } - - var flags int + var f *os.File + var err error if createSize >= 0 { - flags = os.O_CREATE | os.O_TRUNC | os.O_WRONLY - } else { - flags = os.O_WRONLY - } - - f, err := os.OpenFile(path, flags, 0600) - if err != nil { + if f, err = os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600); err != nil { + if fs.IsAccessDenied(err) { + // If file is readonly, clear the readonly flag by resetting the + // permissions of the file and try again + // as the metadata will be set again in the second pass and the + // readonly flag will be applied again if needed. + if err = fs.ResetPermissions(path); err != nil { + return nil, err + } + if f, err = os.OpenFile(path, os.O_TRUNC|os.O_WRONLY, 0600); err != nil { + return nil, err + } + } else { + return nil, err + } + } + } else if f, err = os.OpenFile(path, os.O_WRONLY, 0600); err != nil { return nil, err } From eeb1aa5388693c8e6b693cb3941045588080ee69 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia <99904+aneesh-n@users.noreply.github.com> Date: Thu, 22 Feb 2024 17:52:26 -0700 Subject: [PATCH 155/158] Add ability to report warnings to terminal Report warnings to terminal when unrecognized generic attributes are found in the repository. --- cmd/restic/cmd_restore.go | 3 +++ internal/restic/node.go | 20 ++++++++++---------- internal/restic/node_aix.go | 4 ++-- internal/restic/node_netbsd.go | 4 ++-- internal/restic/node_openbsd.go | 4 ++-- internal/restic/node_test.go | 2 +- internal/restic/node_windows.go | 4 ++-- internal/restic/node_xattr.go | 4 ++-- internal/restorer/restorer.go | 3 ++- 9 files changed, 26 insertions(+), 22 deletions(-) diff --git a/cmd/restic/cmd_restore.go b/cmd/restic/cmd_restore.go index 37d304672..58f257541 100644 --- a/cmd/restic/cmd_restore.go +++ b/cmd/restic/cmd_restore.go @@ -178,6 +178,9 @@ func runRestore(ctx context.Context, opts RestoreOptions, gopts GlobalOptions, totalErrors++ return nil } + res.Warn = func(message string) { + msg.E("Warning: %s\n", message) + } excludePatterns := filter.ParsePatterns(opts.Exclude) insensitiveExcludePatterns := filter.ParsePatterns(opts.InsensitiveExclude) diff --git a/internal/restic/node.go b/internal/restic/node.go index 44ca52b0c..cbe9ef363 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -224,8 +224,8 @@ func (node *Node) CreateAt(ctx context.Context, path string, repo BlobLoader) er } // RestoreMetadata restores node metadata -func (node Node) RestoreMetadata(path string) error { - err := node.restoreMetadata(path) +func (node Node) RestoreMetadata(path string, warn func(msg string)) error { + err := node.restoreMetadata(path, warn) if err != nil { debug.Log("restoreMetadata(%s) error %v", path, err) } @@ -233,7 +233,7 @@ func (node Node) RestoreMetadata(path string) error { return err } -func (node Node) restoreMetadata(path string) error { +func (node Node) restoreMetadata(path string, warn func(msg string)) error { var firsterr error if err := lchown(path, int(node.UID), int(node.GID)); err != nil { @@ -261,7 +261,7 @@ func (node Node) restoreMetadata(path string) error { } } - if err := node.restoreGenericAttributes(path); err != nil { + if err := node.restoreGenericAttributes(path, warn); err != nil { debug.Log("error restoring generic attributes for %v: %v", path, err) if firsterr != nil { firsterr = err @@ -766,14 +766,14 @@ func (node *Node) fillTimes(stat *statT) { } // HandleUnknownGenericAttributesFound is used for handling and distinguing between scenarios related to future versions and cross-OS repositories -func HandleUnknownGenericAttributesFound(unknownAttribs []GenericAttributeType) { +func HandleUnknownGenericAttributesFound(unknownAttribs []GenericAttributeType, warn func(msg string)) { for _, unknownAttrib := range unknownAttribs { - handleUnknownGenericAttributeFound(unknownAttrib) + handleUnknownGenericAttributeFound(unknownAttrib, warn) } } // handleUnknownGenericAttributeFound is used for handling and distinguing between scenarios related to future versions and cross-OS repositories -func handleUnknownGenericAttributeFound(genericAttributeType GenericAttributeType) { +func handleUnknownGenericAttributeFound(genericAttributeType GenericAttributeType, warn func(msg string)) { if checkGenericAttributeNameNotHandledAndPut(genericAttributeType) { // Print the unique error only once for a given execution os, exists := genericAttributesForOS[genericAttributeType] @@ -784,7 +784,7 @@ func handleUnknownGenericAttributeFound(genericAttributeType GenericAttributeTyp debug.Log("Ignoring a generic attribute found in the repository: %s which may not be compatible with your OS. Compatible OS: %s", genericAttributeType, os) } else { // If genericAttributesForOS in node.go does not know about this attribute, then the repository may have been created by a newer version which has a newer GenericAttributeType. - debug.Log("Found an unrecognized generic attribute in the repository: %s. You may need to upgrade to latest version of restic.", genericAttributeType) + warn(fmt.Sprintf("Found an unrecognized generic attribute in the repository: %s. You may need to upgrade to latest version of restic.", genericAttributeType)) } } } @@ -792,9 +792,9 @@ func handleUnknownGenericAttributeFound(genericAttributeType GenericAttributeTyp // handleAllUnknownGenericAttributesFound performs validations for all generic attributes in the node. // This is not used on windows currently because windows has handling for generic attributes. // nolint:unused -func (node Node) handleAllUnknownGenericAttributesFound() error { +func (node Node) handleAllUnknownGenericAttributesFound(warn func(msg string)) error { for name := range node.GenericAttributes { - handleUnknownGenericAttributeFound(name) + handleUnknownGenericAttributeFound(name, warn) } return nil } diff --git a/internal/restic/node_aix.go b/internal/restic/node_aix.go index 4d8c248de..def46bd60 100644 --- a/internal/restic/node_aix.go +++ b/internal/restic/node_aix.go @@ -39,8 +39,8 @@ func Setxattr(path, name string, data []byte) error { } // restoreGenericAttributes is no-op on AIX. -func (node *Node) restoreGenericAttributes(_ string) error { - return node.handleAllUnknownGenericAttributesFound() +func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { + return node.handleAllUnknownGenericAttributesFound(warn) } // fillGenericAttributes is a no-op on AIX. diff --git a/internal/restic/node_netbsd.go b/internal/restic/node_netbsd.go index be4afa3ae..1a47299be 100644 --- a/internal/restic/node_netbsd.go +++ b/internal/restic/node_netbsd.go @@ -29,8 +29,8 @@ func Setxattr(path, name string, data []byte) error { } // restoreGenericAttributes is no-op on netbsd. -func (node *Node) restoreGenericAttributes(_ string) error { - return node.handleAllUnknownGenericAttributesFound() +func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { + return node.handleAllUnknownGenericAttributesFound(warn) } // fillGenericAttributes is a no-op on netbsd. diff --git a/internal/restic/node_openbsd.go b/internal/restic/node_openbsd.go index bfff8f8aa..e60eb9dc8 100644 --- a/internal/restic/node_openbsd.go +++ b/internal/restic/node_openbsd.go @@ -29,8 +29,8 @@ func Setxattr(path, name string, data []byte) error { } // restoreGenericAttributes is no-op on openbsd. -func (node *Node) restoreGenericAttributes(_ string) error { - return node.handleAllUnknownGenericAttributesFound() +func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { + return node.handleAllUnknownGenericAttributesFound(warn) } // fillGenericAttributes is a no-op on openbsd. diff --git a/internal/restic/node_test.go b/internal/restic/node_test.go index c2c7306b7..d9fa02ac8 100644 --- a/internal/restic/node_test.go +++ b/internal/restic/node_test.go @@ -218,7 +218,7 @@ func TestNodeRestoreAt(t *testing.T) { nodePath = filepath.Join(tempdir, test.Name) } rtest.OK(t, test.CreateAt(context.TODO(), nodePath, nil)) - rtest.OK(t, test.RestoreMetadata(nodePath)) + rtest.OK(t, test.RestoreMetadata(nodePath, func(msg string) { rtest.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", nodePath, msg)) })) if test.Type == "dir" { rtest.OK(t, test.RestoreTimestamps(nodePath)) diff --git a/internal/restic/node_windows.go b/internal/restic/node_windows.go index a2b8c75e5..5875c3ccd 100644 --- a/internal/restic/node_windows.go +++ b/internal/restic/node_windows.go @@ -118,7 +118,7 @@ func (s statT) ctim() syscall.Timespec { } // restoreGenericAttributes restores generic attributes for Windows -func (node Node) restoreGenericAttributes(path string) (err error) { +func (node Node) restoreGenericAttributes(path string, warn func(msg string)) (err error) { if len(node.GenericAttributes) == 0 { return nil } @@ -138,7 +138,7 @@ func (node Node) restoreGenericAttributes(path string) (err error) { } } - HandleUnknownGenericAttributesFound(unknownAttribs) + HandleUnknownGenericAttributesFound(unknownAttribs, warn) return errors.CombineErrors(errs...) } diff --git a/internal/restic/node_xattr.go b/internal/restic/node_xattr.go index 826b8b74a..0b2d5d552 100644 --- a/internal/restic/node_xattr.go +++ b/internal/restic/node_xattr.go @@ -50,8 +50,8 @@ func handleXattrErr(err error) error { } // restoreGenericAttributes is no-op. -func (node *Node) restoreGenericAttributes(_ string) error { - return node.handleAllUnknownGenericAttributesFound() +func (node *Node) restoreGenericAttributes(_ string, warn func(msg string)) error { + return node.handleAllUnknownGenericAttributesFound(warn) } // fillGenericAttributes is a no-op. diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 3f4fb32e3..0aeb636d0 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -24,6 +24,7 @@ type Restorer struct { progress *restoreui.Progress Error func(location string, err error) error + Warn func(message string) SelectFilter func(item string, dstpath string, node *restic.Node) (selectedForRestore bool, childMayBeSelected bool) } @@ -178,7 +179,7 @@ func (res *Restorer) restoreNodeTo(ctx context.Context, node *restic.Node, targe func (res *Restorer) restoreNodeMetadataTo(node *restic.Node, target, location string) error { debug.Log("restoreNodeMetadata %v %v %v", node.Name, target, location) - err := node.RestoreMetadata(target) + err := node.RestoreMetadata(target, res.Warn) if err != nil { debug.Log("node.RestoreMetadata(%s) error %v", target, err) } From d4be734c73565f822a4965c835bf302330631464 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia <99904+aneesh-n@users.noreply.github.com> Date: Thu, 22 Feb 2024 17:54:43 -0700 Subject: [PATCH 156/158] Handle readonly empty files in windows --- internal/restorer/restorer.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/internal/restorer/restorer.go b/internal/restorer/restorer.go index 0aeb636d0..9f41f5cf2 100644 --- a/internal/restorer/restorer.go +++ b/internal/restorer/restorer.go @@ -205,11 +205,19 @@ func (res *Restorer) restoreHardlinkAt(node *restic.Node, target, path, location func (res *Restorer) restoreEmptyFileAt(node *restic.Node, target, location string) error { wr, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) - if err != nil { - return err + if fs.IsAccessDenied(err) { + // If file is readonly, clear the readonly flag by resetting the + // permissions of the file and try again + // as the metadata will be set again in the second pass and the + // readonly flag will be applied again if needed. + if err = fs.ResetPermissions(target); err != nil { + return err + } + if wr, err = os.OpenFile(target, os.O_TRUNC|os.O_WRONLY, 0600); err != nil { + return err + } } - err = wr.Close() - if err != nil { + if err = wr.Close(); err != nil { return err } From 4bbd25a37fb4554b01af7b567ac7b4f16147a588 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia <99904+aneesh-n@users.noreply.github.com> Date: Thu, 22 Feb 2024 17:55:50 -0700 Subject: [PATCH 157/158] Add tests for generic attribute changes --- internal/restic/node_windows_test.go | 210 +++++++++ internal/restorer/restorer_test.go | 86 ++-- internal/restorer/restorer_unix_test.go | 4 +- internal/restorer/restorer_windows_test.go | 506 +++++++++++++++++++++ 4 files changed, 770 insertions(+), 36 deletions(-) create mode 100644 internal/restic/node_windows_test.go diff --git a/internal/restic/node_windows_test.go b/internal/restic/node_windows_test.go new file mode 100644 index 000000000..501d5a98a --- /dev/null +++ b/internal/restic/node_windows_test.go @@ -0,0 +1,210 @@ +//go:build windows +// +build windows + +package restic + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "syscall" + "testing" + + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/test" + "golang.org/x/sys/windows" +) + +func TestRestoreCreationTime(t *testing.T) { + t.Parallel() + path := t.TempDir() + fi, err := os.Lstat(path) + test.OK(t, errors.Wrapf(err, "Could not Lstat for path: %s", path)) + creationTimeAttribute := getCreationTime(fi, path) + test.OK(t, errors.Wrapf(err, "Could not get creation time for path: %s", path)) + //Using the temp dir creation time as the test creation time for the test file and folder + runGenericAttributesTest(t, path, TypeCreationTime, WindowsAttributes{CreationTime: creationTimeAttribute}, false) +} + +func TestRestoreFileAttributes(t *testing.T) { + t.Parallel() + genericAttributeName := TypeFileAttributes + tempDir := t.TempDir() + normal := uint32(syscall.FILE_ATTRIBUTE_NORMAL) + hidden := uint32(syscall.FILE_ATTRIBUTE_HIDDEN) + system := uint32(syscall.FILE_ATTRIBUTE_SYSTEM) + archive := uint32(syscall.FILE_ATTRIBUTE_ARCHIVE) + encrypted := uint32(windows.FILE_ATTRIBUTE_ENCRYPTED) + fileAttributes := []WindowsAttributes{ + //normal + {FileAttributes: &normal}, + //hidden + {FileAttributes: &hidden}, + //system + {FileAttributes: &system}, + //archive + {FileAttributes: &archive}, + //encrypted + {FileAttributes: &encrypted}, + } + for i, fileAttr := range fileAttributes { + genericAttrs, err := WindowsAttrsToGenericAttributes(fileAttr) + test.OK(t, err) + expectedNodes := []Node{ + { + Name: fmt.Sprintf("testfile%d", i), + Type: "file", + Mode: 0655, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: genericAttrs, + }, + } + runGenericAttributesTestForNodes(t, expectedNodes, tempDir, genericAttributeName, fileAttr, false) + } + normal = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY) + hidden = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | syscall.FILE_ATTRIBUTE_HIDDEN) + system = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | windows.FILE_ATTRIBUTE_SYSTEM) + archive = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | windows.FILE_ATTRIBUTE_ARCHIVE) + encrypted = uint32(syscall.FILE_ATTRIBUTE_DIRECTORY | windows.FILE_ATTRIBUTE_ENCRYPTED) + folderAttributes := []WindowsAttributes{ + //normal + {FileAttributes: &normal}, + //hidden + {FileAttributes: &hidden}, + //system + {FileAttributes: &system}, + //archive + {FileAttributes: &archive}, + //encrypted + {FileAttributes: &encrypted}, + } + for i, folderAttr := range folderAttributes { + genericAttrs, err := WindowsAttrsToGenericAttributes(folderAttr) + test.OK(t, err) + expectedNodes := []Node{ + { + Name: fmt.Sprintf("testdirectory%d", i), + Type: "dir", + Mode: 0755, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: genericAttrs, + }, + } + runGenericAttributesTestForNodes(t, expectedNodes, tempDir, genericAttributeName, folderAttr, false) + } +} + +func runGenericAttributesTest(t *testing.T, tempDir string, genericAttributeName GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { + genericAttributes, err := WindowsAttrsToGenericAttributes(genericAttributeExpected) + test.OK(t, err) + expectedNodes := []Node{ + { + Name: "testfile", + Type: "file", + Mode: 0644, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: genericAttributes, + }, + { + Name: "testdirectory", + Type: "dir", + Mode: 0755, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: genericAttributes, + }, + } + runGenericAttributesTestForNodes(t, expectedNodes, tempDir, genericAttributeName, genericAttributeExpected, warningExpected) +} +func runGenericAttributesTestForNodes(t *testing.T, expectedNodes []Node, tempDir string, genericAttr GenericAttributeType, genericAttributeExpected WindowsAttributes, warningExpected bool) { + + for _, testNode := range expectedNodes { + testPath, node := restoreAndGetNode(t, tempDir, testNode, warningExpected) + rawMessage := node.GenericAttributes[genericAttr] + genericAttrsExpected, err := WindowsAttrsToGenericAttributes(genericAttributeExpected) + test.OK(t, err) + rawMessageExpected := genericAttrsExpected[genericAttr] + test.Equals(t, rawMessageExpected, rawMessage, "Generic attribute: %s got from NodeFromFileInfo not equal for path: %s", string(genericAttr), testPath) + } +} + +func restoreAndGetNode(t *testing.T, tempDir string, testNode Node, warningExpected bool) (string, *Node) { + testPath := filepath.Join(tempDir, "001", testNode.Name) + err := os.MkdirAll(filepath.Dir(testPath), testNode.Mode) + test.OK(t, errors.Wrapf(err, "Failed to create parent directories for: %s", testPath)) + + if testNode.Type == "file" { + + testFile, err := os.Create(testPath) + test.OK(t, errors.Wrapf(err, "Failed to create test file: %s", testPath)) + testFile.Close() + } else if testNode.Type == "dir" { + + err := os.Mkdir(testPath, testNode.Mode) + test.OK(t, errors.Wrapf(err, "Failed to create test directory: %s", testPath)) + } + + err = testNode.RestoreMetadata(testPath, func(msg string) { + if warningExpected { + test.Assert(t, warningExpected, "Warning triggered as expected: %s", msg) + } else { + // If warning is not expected, this code should not get triggered. + test.OK(t, fmt.Errorf("Warning triggered for path: %s: %s", testPath, msg)) + } + }) + test.OK(t, errors.Wrapf(err, "Failed to restore metadata for: %s", testPath)) + + fi, err := os.Lstat(testPath) + test.OK(t, errors.Wrapf(err, "Could not Lstat for path: %s", testPath)) + + nodeFromFileInfo, err := NodeFromFileInfo(testPath, fi) + test.OK(t, errors.Wrapf(err, "Could not get NodeFromFileInfo for path: %s", testPath)) + + return testPath, nodeFromFileInfo +} + +const TypeSomeNewAttribute GenericAttributeType = "MockAttributes.SomeNewAttribute" + +func TestNewGenericAttributeType(t *testing.T) { + t.Parallel() + + newGenericAttribute := map[GenericAttributeType]json.RawMessage{} + newGenericAttribute[TypeSomeNewAttribute] = []byte("any value") + + tempDir := t.TempDir() + expectedNodes := []Node{ + { + Name: "testfile", + Type: "file", + Mode: 0644, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: newGenericAttribute, + }, + { + Name: "testdirectory", + Type: "dir", + Mode: 0755, + ModTime: parseTime("2005-05-14 21:07:03.111"), + AccessTime: parseTime("2005-05-14 21:07:04.222"), + ChangeTime: parseTime("2005-05-14 21:07:05.333"), + GenericAttributes: newGenericAttribute, + }, + } + for _, testNode := range expectedNodes { + testPath, node := restoreAndGetNode(t, tempDir, testNode, true) + _, ua, err := genericAttributesToWindowsAttrs(node.GenericAttributes) + test.OK(t, err) + // Since this GenericAttribute is unknown to this version of the software, it will not get set on the file. + test.Assert(t, len(ua) == 0, "Unkown attributes: %s found for path: %s", ua, testPath) + } +} diff --git a/internal/restorer/restorer_test.go b/internal/restorer/restorer_test.go index c33214bc3..5742d7663 100644 --- a/internal/restorer/restorer_test.go +++ b/internal/restorer/restorer_test.go @@ -3,6 +3,7 @@ package restorer import ( "bytes" "context" + "encoding/json" "io" "math" "os" @@ -27,17 +28,27 @@ type Snapshot struct { } type File struct { - Data string - Links uint64 - Inode uint64 - Mode os.FileMode - ModTime time.Time + Data string + Links uint64 + Inode uint64 + Mode os.FileMode + ModTime time.Time + attributes *FileAttributes } type Dir struct { - Nodes map[string]Node - Mode os.FileMode - ModTime time.Time + Nodes map[string]Node + Mode os.FileMode + ModTime time.Time + attributes *FileAttributes +} + +type FileAttributes struct { + ReadOnly bool + Hidden bool + System bool + Archive bool + Encrypted bool } func saveFile(t testing.TB, repo restic.BlobSaver, node File) restic.ID { @@ -52,7 +63,7 @@ func saveFile(t testing.TB, repo restic.BlobSaver, node File) restic.ID { return id } -func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode uint64) restic.ID { +func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode uint64, getGenericAttributes func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage)) restic.ID { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -78,20 +89,21 @@ func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode u mode = 0644 } err := tree.Insert(&restic.Node{ - Type: "file", - Mode: mode, - ModTime: node.ModTime, - Name: name, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Content: fc, - Size: uint64(len(n.(File).Data)), - Inode: fi, - Links: lc, + Type: "file", + Mode: mode, + ModTime: node.ModTime, + Name: name, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Content: fc, + Size: uint64(len(n.(File).Data)), + Inode: fi, + Links: lc, + GenericAttributes: getGenericAttributes(node.attributes, false), }) rtest.OK(t, err) case Dir: - id := saveDir(t, repo, node.Nodes, inode) + id := saveDir(t, repo, node.Nodes, inode, getGenericAttributes) mode := node.Mode if mode == 0 { @@ -99,13 +111,14 @@ func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode u } err := tree.Insert(&restic.Node{ - Type: "dir", - Mode: mode, - ModTime: node.ModTime, - Name: name, - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - Subtree: &id, + Type: "dir", + Mode: mode, + ModTime: node.ModTime, + Name: name, + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + Subtree: &id, + GenericAttributes: getGenericAttributes(node.attributes, false), }) rtest.OK(t, err) default: @@ -121,13 +134,13 @@ func saveDir(t testing.TB, repo restic.BlobSaver, nodes map[string]Node, inode u return id } -func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot) (*restic.Snapshot, restic.ID) { +func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot, getGenericAttributes func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage)) (*restic.Snapshot, restic.ID) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() wg, wgCtx := errgroup.WithContext(ctx) repo.StartPackUploader(wgCtx, wg) - treeID := saveDir(t, repo, snapshot.Nodes, 1000) + treeID := saveDir(t, repo, snapshot.Nodes, 1000, getGenericAttributes) err := repo.Flush(ctx) if err != nil { t.Fatal(err) @@ -147,6 +160,11 @@ func saveSnapshot(t testing.TB, repo restic.Repository, snapshot Snapshot) (*res return sn, id } +var noopGetGenericAttributes = func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage) { + // No-op + return nil +} + func TestRestorer(t *testing.T) { var tests = []struct { Snapshot @@ -322,7 +340,7 @@ func TestRestorer(t *testing.T) { for _, test := range tests { t.Run("", func(t *testing.T) { repo := repository.TestRepository(t) - sn, id := saveSnapshot(t, repo, test.Snapshot) + sn, id := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes) t.Logf("snapshot saved as %v", id.Str()) res := NewRestorer(repo, sn, false, nil) @@ -439,7 +457,7 @@ func TestRestorerRelative(t *testing.T) { t.Run("", func(t *testing.T) { repo := repository.TestRepository(t) - sn, id := saveSnapshot(t, repo, test.Snapshot) + sn, id := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes) t.Logf("snapshot saved as %v", id.Str()) res := NewRestorer(repo, sn, false, nil) @@ -669,7 +687,7 @@ func TestRestorerTraverseTree(t *testing.T) { for _, test := range tests { t.Run("", func(t *testing.T) { repo := repository.TestRepository(t) - sn, _ := saveSnapshot(t, repo, test.Snapshot) + sn, _ := saveSnapshot(t, repo, test.Snapshot, noopGetGenericAttributes) res := NewRestorer(repo, sn, false, nil) @@ -745,7 +763,7 @@ func TestRestorerConsistentTimestampsAndPermissions(t *testing.T) { }, }, }, - }) + }, noopGetGenericAttributes) res := NewRestorer(repo, sn, false, nil) @@ -800,7 +818,7 @@ func TestVerifyCancel(t *testing.T) { } repo := repository.TestRepository(t) - sn, _ := saveSnapshot(t, repo, snapshot) + sn, _ := saveSnapshot(t, repo, snapshot, noopGetGenericAttributes) res := NewRestorer(repo, sn, false, nil) diff --git a/internal/restorer/restorer_unix_test.go b/internal/restorer/restorer_unix_test.go index 2c30a6b64..0cbfefa92 100644 --- a/internal/restorer/restorer_unix_test.go +++ b/internal/restorer/restorer_unix_test.go @@ -29,7 +29,7 @@ func TestRestorerRestoreEmptyHardlinkedFileds(t *testing.T) { }, }, }, - }) + }, noopGetGenericAttributes) res := NewRestorer(repo, sn, false, nil) @@ -95,7 +95,7 @@ func TestRestorerProgressBar(t *testing.T) { }, "file2": File{Links: 1, Inode: 2, Data: "example"}, }, - }) + }, noopGetGenericAttributes) mock := &printerMock{} progress := restoreui.NewProgress(mock, 0) diff --git a/internal/restorer/restorer_windows_test.go b/internal/restorer/restorer_windows_test.go index 3ec4b1f11..684d51ace 100644 --- a/internal/restorer/restorer_windows_test.go +++ b/internal/restorer/restorer_windows_test.go @@ -4,11 +4,20 @@ package restorer import ( + "context" + "encoding/json" "math" + "os" + "path" "syscall" "testing" + "time" "unsafe" + "github.com/restic/restic/internal/errors" + "github.com/restic/restic/internal/repository" + "github.com/restic/restic/internal/restic" + "github.com/restic/restic/internal/test" rtest "github.com/restic/restic/internal/test" "golang.org/x/sys/windows" ) @@ -33,3 +42,500 @@ func getBlockCount(t *testing.T, filename string) int64 { return int64(math.Ceil(float64(result) / 512)) } + +type DataStreamInfo struct { + name string + data string +} + +type NodeInfo struct { + DataStreamInfo + parentDir string + attributes FileAttributes + Exists bool + IsDirectory bool +} + +func TestFileAttributeCombination(t *testing.T) { + testFileAttributeCombination(t, false) +} + +func TestEmptyFileAttributeCombination(t *testing.T) { + testFileAttributeCombination(t, true) +} + +func testFileAttributeCombination(t *testing.T, isEmpty bool) { + t.Parallel() + //Generate combination of 5 attributes. + attributeCombinations := generateCombinations(5, []bool{}) + + fileName := "TestFile.txt" + // Iterate through each attribute combination + for _, attr1 := range attributeCombinations { + + //Set up the required file information + fileInfo := NodeInfo{ + DataStreamInfo: getDataStreamInfo(isEmpty, fileName), + parentDir: "dir", + attributes: getFileAttributes(attr1), + Exists: false, + } + + //Get the current test name + testName := getCombinationTestName(fileInfo, fileName, fileInfo.attributes) + + //Run test + t.Run(testName, func(t *testing.T) { + mainFilePath := runAttributeTests(t, fileInfo, fileInfo.attributes) + + verifyFileRestores(isEmpty, mainFilePath, t, fileInfo) + }) + } +} + +func generateCombinations(n int, prefix []bool) [][]bool { + if n == 0 { + // Return a slice containing the current permutation + return [][]bool{append([]bool{}, prefix...)} + } + + // Generate combinations with True + prefixTrue := append(prefix, true) + permsTrue := generateCombinations(n-1, prefixTrue) + + // Generate combinations with False + prefixFalse := append(prefix, false) + permsFalse := generateCombinations(n-1, prefixFalse) + + // Combine combinations with True and False + return append(permsTrue, permsFalse...) +} + +func getDataStreamInfo(isEmpty bool, fileName string) DataStreamInfo { + var dataStreamInfo DataStreamInfo + if isEmpty { + dataStreamInfo = DataStreamInfo{ + name: fileName, + } + } else { + dataStreamInfo = DataStreamInfo{ + name: fileName, + data: "Main file data stream.", + } + } + return dataStreamInfo +} + +func getFileAttributes(values []bool) FileAttributes { + return FileAttributes{ + ReadOnly: values[0], + Hidden: values[1], + System: values[2], + Archive: values[3], + Encrypted: values[4], + } +} + +func getCombinationTestName(fi NodeInfo, fileName string, overwriteAttr FileAttributes) string { + if fi.attributes.ReadOnly { + fileName += "-ReadOnly" + } + if fi.attributes.Hidden { + fileName += "-Hidden" + } + if fi.attributes.System { + fileName += "-System" + } + if fi.attributes.Archive { + fileName += "-Archive" + } + if fi.attributes.Encrypted { + fileName += "-Encrypted" + } + if fi.Exists { + fileName += "-Overwrite" + if overwriteAttr.ReadOnly { + fileName += "-R" + } + if overwriteAttr.Hidden { + fileName += "-H" + } + if overwriteAttr.System { + fileName += "-S" + } + if overwriteAttr.Archive { + fileName += "-A" + } + if overwriteAttr.Encrypted { + fileName += "-E" + } + } + return fileName +} + +func runAttributeTests(t *testing.T, fileInfo NodeInfo, existingFileAttr FileAttributes) string { + testDir := t.TempDir() + res, _ := setupWithFileAttributes(t, fileInfo, testDir, existingFileAttr) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := res.RestoreTo(ctx, testDir) + rtest.OK(t, err) + + mainFilePath := path.Join(testDir, fileInfo.parentDir, fileInfo.name) + //Verify restore + verifyFileAttributes(t, mainFilePath, fileInfo.attributes) + return mainFilePath +} + +func setupWithFileAttributes(t *testing.T, nodeInfo NodeInfo, testDir string, existingFileAttr FileAttributes) (*Restorer, []int) { + t.Helper() + if nodeInfo.Exists { + if !nodeInfo.IsDirectory { + err := os.MkdirAll(path.Join(testDir, nodeInfo.parentDir), os.ModeDir) + rtest.OK(t, err) + filepath := path.Join(testDir, nodeInfo.parentDir, nodeInfo.name) + if existingFileAttr.Encrypted { + err := createEncryptedFileWriteData(filepath, nodeInfo) + rtest.OK(t, err) + } else { + // Write the data to the file + file, err := os.OpenFile(path.Clean(filepath), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + rtest.OK(t, err) + _, err = file.Write([]byte(nodeInfo.data)) + rtest.OK(t, err) + + err = file.Close() + rtest.OK(t, err) + } + } else { + err := os.MkdirAll(path.Join(testDir, nodeInfo.parentDir, nodeInfo.name), os.ModeDir) + rtest.OK(t, err) + } + + pathPointer, err := syscall.UTF16PtrFromString(path.Join(testDir, nodeInfo.parentDir, nodeInfo.name)) + rtest.OK(t, err) + syscall.SetFileAttributes(pathPointer, getAttributeValue(&existingFileAttr)) + } + + index := 0 + + order := []int{} + streams := []DataStreamInfo{} + if !nodeInfo.IsDirectory { + order = append(order, index) + index++ + streams = append(streams, nodeInfo.DataStreamInfo) + } + return setup(t, getNodes(nodeInfo.parentDir, nodeInfo.name, order, streams, nodeInfo.IsDirectory, &nodeInfo.attributes)), order +} + +func createEncryptedFileWriteData(filepath string, fileInfo NodeInfo) (err error) { + var ptr *uint16 + if ptr, err = windows.UTF16PtrFromString(filepath); err != nil { + return err + } + var handle windows.Handle + //Create the file with encrypted flag + if handle, err = windows.CreateFile(ptr, uint32(windows.GENERIC_READ|windows.GENERIC_WRITE), uint32(windows.FILE_SHARE_READ), nil, uint32(windows.CREATE_ALWAYS), windows.FILE_ATTRIBUTE_ENCRYPTED, 0); err != nil { + return err + } + //Write data to file + if _, err = windows.Write(handle, []byte(fileInfo.data)); err != nil { + return err + } + //Close handle + return windows.CloseHandle(handle) +} + +func setup(t *testing.T, nodesMap map[string]Node) *Restorer { + repo := repository.TestRepository(t) + getFileAttributes := func(attr *FileAttributes, isDir bool) (genericAttributes map[restic.GenericAttributeType]json.RawMessage) { + if attr == nil { + return + } + + fileattr := getAttributeValue(attr) + + if isDir { + //If the node is a directory add FILE_ATTRIBUTE_DIRECTORY to attributes + fileattr |= windows.FILE_ATTRIBUTE_DIRECTORY + } + attrs, err := restic.WindowsAttrsToGenericAttributes(restic.WindowsAttributes{FileAttributes: &fileattr}) + test.OK(t, err) + return attrs + } + sn, _ := saveSnapshot(t, repo, Snapshot{ + Nodes: nodesMap, + }, getFileAttributes) + res := NewRestorer(repo, sn, false, nil) + return res +} + +func getAttributeValue(attr *FileAttributes) uint32 { + var fileattr uint32 + if attr.ReadOnly { + fileattr |= windows.FILE_ATTRIBUTE_READONLY + } + if attr.Hidden { + fileattr |= windows.FILE_ATTRIBUTE_HIDDEN + } + if attr.Encrypted { + fileattr |= windows.FILE_ATTRIBUTE_ENCRYPTED + } + if attr.Archive { + fileattr |= windows.FILE_ATTRIBUTE_ARCHIVE + } + if attr.System { + fileattr |= windows.FILE_ATTRIBUTE_SYSTEM + } + return fileattr +} + +func getNodes(dir string, mainNodeName string, order []int, streams []DataStreamInfo, isDirectory bool, attributes *FileAttributes) map[string]Node { + var mode os.FileMode + if isDirectory { + mode = os.FileMode(2147484159) + } else { + if attributes != nil && attributes.ReadOnly { + mode = os.FileMode(0o444) + } else { + mode = os.FileMode(0o666) + } + } + + getFileNodes := func() map[string]Node { + nodes := map[string]Node{} + if isDirectory { + //Add a directory node at the same level as the other streams + nodes[mainNodeName] = Dir{ + ModTime: time.Now(), + attributes: attributes, + Mode: mode, + } + } + + if len(streams) > 0 { + for _, index := range order { + stream := streams[index] + + var attr *FileAttributes = nil + if mainNodeName == stream.name { + attr = attributes + } else if attributes != nil && attributes.Encrypted { + //Set encrypted attribute + attr = &FileAttributes{Encrypted: true} + } + + nodes[stream.name] = File{ + ModTime: time.Now(), + Data: stream.data, + Mode: mode, + attributes: attr, + } + } + } + return nodes + } + + return map[string]Node{ + dir: Dir{ + Mode: normalizeFileMode(0750 | mode), + ModTime: time.Now(), + Nodes: getFileNodes(), + }, + } +} + +func verifyFileAttributes(t *testing.T, mainFilePath string, attr FileAttributes) { + ptr, err := windows.UTF16PtrFromString(mainFilePath) + rtest.OK(t, err) + //Get file attributes using syscall + fileAttributes, err := syscall.GetFileAttributes(ptr) + rtest.OK(t, err) + //Test positive and negative scenarios + if attr.ReadOnly { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_READONLY != 0, "Expected read only attibute.") + } else { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_READONLY == 0, "Unexpected read only attibute.") + } + if attr.Hidden { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_HIDDEN != 0, "Expected hidden attibute.") + } else { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_HIDDEN == 0, "Unexpected hidden attibute.") + } + if attr.System { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_SYSTEM != 0, "Expected system attibute.") + } else { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_SYSTEM == 0, "Unexpected system attibute.") + } + if attr.Archive { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_ARCHIVE != 0, "Expected archive attibute.") + } else { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_ARCHIVE == 0, "Unexpected archive attibute.") + } + if attr.Encrypted { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_ENCRYPTED != 0, "Expected encrypted attibute.") + } else { + rtest.Assert(t, fileAttributes&windows.FILE_ATTRIBUTE_ENCRYPTED == 0, "Unexpected encrypted attibute.") + } +} + +func verifyFileRestores(isEmpty bool, mainFilePath string, t *testing.T, fileInfo NodeInfo) { + if isEmpty { + _, err1 := os.Stat(mainFilePath) + rtest.Assert(t, !errors.Is(err1, os.ErrNotExist), "The file "+fileInfo.name+" does not exist") + } else { + + verifyMainFileRestore(t, mainFilePath, fileInfo) + } +} + +func verifyMainFileRestore(t *testing.T, mainFilePath string, fileInfo NodeInfo) { + fi, err1 := os.Stat(mainFilePath) + rtest.Assert(t, !errors.Is(err1, os.ErrNotExist), "The file "+fileInfo.name+" does not exist") + + size := fi.Size() + rtest.Assert(t, size > 0, "The file "+fileInfo.name+" exists but is empty") + + content, err := os.ReadFile(mainFilePath) + rtest.OK(t, err) + rtest.Assert(t, string(content) == fileInfo.data, "The file "+fileInfo.name+" exists but the content is not overwritten") +} + +func TestDirAttributeCombination(t *testing.T) { + t.Parallel() + attributeCombinations := generateCombinations(4, []bool{}) + + dirName := "TestDir" + // Iterate through each attribute combination + for _, attr1 := range attributeCombinations { + + //Set up the required directory information + dirInfo := NodeInfo{ + DataStreamInfo: DataStreamInfo{ + name: dirName, + }, + parentDir: "dir", + attributes: getDirFileAttributes(attr1), + Exists: false, + IsDirectory: true, + } + + //Get the current test name + testName := getCombinationTestName(dirInfo, dirName, dirInfo.attributes) + + //Run test + t.Run(testName, func(t *testing.T) { + mainDirPath := runAttributeTests(t, dirInfo, dirInfo.attributes) + + //Check directory exists + _, err1 := os.Stat(mainDirPath) + rtest.Assert(t, !errors.Is(err1, os.ErrNotExist), "The directory "+dirInfo.name+" does not exist") + }) + } +} + +func getDirFileAttributes(values []bool) FileAttributes { + return FileAttributes{ + // readonly not valid for directories + Hidden: values[0], + System: values[1], + Archive: values[2], + Encrypted: values[3], + } +} + +func TestFileAttributeCombinationsOverwrite(t *testing.T) { + testFileAttributeCombinationsOverwrite(t, false) +} + +func TestEmptyFileAttributeCombinationsOverwrite(t *testing.T) { + testFileAttributeCombinationsOverwrite(t, true) +} + +func testFileAttributeCombinationsOverwrite(t *testing.T, isEmpty bool) { + t.Parallel() + //Get attribute combinations + attributeCombinations := generateCombinations(5, []bool{}) + //Get overwrite file attribute combinations + overwriteCombinations := generateCombinations(5, []bool{}) + + fileName := "TestOverwriteFile" + + //Iterate through each attribute combination + for _, attr1 := range attributeCombinations { + + fileInfo := NodeInfo{ + DataStreamInfo: getDataStreamInfo(isEmpty, fileName), + parentDir: "dir", + attributes: getFileAttributes(attr1), + Exists: true, + } + + overwriteFileAttributes := []FileAttributes{} + + for _, overwrite := range overwriteCombinations { + overwriteFileAttributes = append(overwriteFileAttributes, getFileAttributes(overwrite)) + } + + //Iterate through each overwrite attribute combination + for _, overwriteFileAttr := range overwriteFileAttributes { + //Get the test name + testName := getCombinationTestName(fileInfo, fileName, overwriteFileAttr) + + //Run test + t.Run(testName, func(t *testing.T) { + mainFilePath := runAttributeTests(t, fileInfo, overwriteFileAttr) + + verifyFileRestores(isEmpty, mainFilePath, t, fileInfo) + }) + } + } +} + +func TestDirAttributeCombinationsOverwrite(t *testing.T) { + t.Parallel() + //Get attribute combinations + attributeCombinations := generateCombinations(4, []bool{}) + //Get overwrite dir attribute combinations + overwriteCombinations := generateCombinations(4, []bool{}) + + dirName := "TestOverwriteDir" + + //Iterate through each attribute combination + for _, attr1 := range attributeCombinations { + + dirInfo := NodeInfo{ + DataStreamInfo: DataStreamInfo{ + name: dirName, + }, + parentDir: "dir", + attributes: getDirFileAttributes(attr1), + Exists: true, + IsDirectory: true, + } + + overwriteDirFileAttributes := []FileAttributes{} + + for _, overwrite := range overwriteCombinations { + overwriteDirFileAttributes = append(overwriteDirFileAttributes, getDirFileAttributes(overwrite)) + } + + //Iterate through each overwrite attribute combinations + for _, overwriteDirAttr := range overwriteDirFileAttributes { + //Get the test name + testName := getCombinationTestName(dirInfo, dirName, overwriteDirAttr) + + //Run test + t.Run(testName, func(t *testing.T) { + mainDirPath := runAttributeTests(t, dirInfo, dirInfo.attributes) + + //Check directory exists + _, err1 := os.Stat(mainDirPath) + rtest.Assert(t, !errors.Is(err1, os.ErrNotExist), "The directory "+dirInfo.name+" does not exist") + }) + } + } +} From e8211cb64a80fc4f949b5088c167d51dc4ef45a1 Mon Sep 17 00:00:00 2001 From: Aneesh Nireshwalia <99904+aneesh-n@users.noreply.github.com> Date: Thu, 22 Feb 2024 17:59:56 -0700 Subject: [PATCH 158/158] Add changelog and update docs for windows attr --- changelog/unreleased/pull-4611 | 7 +++++++ doc/040_backup.rst | 1 - 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/pull-4611 diff --git a/changelog/unreleased/pull-4611 b/changelog/unreleased/pull-4611 new file mode 100644 index 000000000..940de9c26 --- /dev/null +++ b/changelog/unreleased/pull-4611 @@ -0,0 +1,7 @@ +Enhancement: Back up windows created time and file attributes like hidden flag + +Restic did not back up windows-specific meta-data like created time and file attributes like hidden flag. +Restic now backs up file created time and file attributes like hidden, readonly and encrypted flag when backing up files and folders on windows. + +https://github.com/restic/restic/pull/4611 + diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 550957eeb..d0bd4b2e2 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -487,7 +487,6 @@ particular note are: * File creation date on Unix platforms * Inode flags on Unix platforms * File ownership and ACLs on Windows -* The "hidden" flag on Windows Reading data from a command ***************************