From 44646c20bed56d8e02c99770ef478b086ef1363c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 01:17:57 +0000 Subject: [PATCH 01/31] build(deps): bump github.com/klauspost/compress from 1.17.2 to 1.17.4 Bumps [github.com/klauspost/compress](https://github.com/klauspost/compress) from 1.17.2 to 1.17.4. - [Release notes](https://github.com/klauspost/compress/releases) - [Changelog](https://github.com/klauspost/compress/blob/master/.goreleaser.yml) - [Commits](https://github.com/klauspost/compress/compare/v1.17.2...v1.17.4) --- updated-dependencies: - dependency-name: github.com/klauspost/compress dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a18fd1fde..a379a3083 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/go-ole/go-ole v1.3.0 github.com/google/go-cmp v0.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 - github.com/klauspost/compress v1.17.2 + github.com/klauspost/compress v1.17.4 github.com/minio/minio-go/v7 v7.0.63 github.com/minio/sha256-simd v1.0.1 github.com/ncw/swift/v2 v2.0.2 diff --git a/go.sum b/go.sum index 61ed82f8b..52faccbe9 100644 --- a/go.sum +++ b/go.sum @@ -108,8 +108,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= -github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= From 7e7cbe8e19290212f7e0e6e3ec6fd82b291bd626 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 01:18:08 +0000 Subject: [PATCH 02/31] build(deps): bump golang.org/x/crypto from 0.16.0 to 0.17.0 Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.16.0 to 0.17.0. - [Commits](https://github.com/golang/crypto/compare/v0.16.0...v0.17.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a18fd1fde..1731b5647 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 go.uber.org/automaxprocs v1.5.3 - golang.org/x/crypto v0.16.0 + golang.org/x/crypto v0.17.0 golang.org/x/net v0.19.0 golang.org/x/oauth2 v0.15.0 golang.org/x/sync v0.5.0 diff --git a/go.sum b/go.sum index 61ed82f8b..55b088e49 100644 --- a/go.sum +++ b/go.sum @@ -183,8 +183,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= -golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= From 40905403f41aa0904a6a5bb8638ed234c33da8fb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 01:18:36 +0000 Subject: [PATCH 03/31] build(deps): bump github.com/Azure/azure-sdk-for-go/sdk/azcore Bumps [github.com/Azure/azure-sdk-for-go/sdk/azcore](https://github.com/Azure/azure-sdk-for-go) from 1.8.0 to 1.9.1. - [Release notes](https://github.com/Azure/azure-sdk-for-go/releases) - [Changelog](https://github.com/Azure/azure-sdk-for-go/blob/main/documentation/release.md) - [Commits](https://github.com/Azure/azure-sdk-for-go/compare/sdk/azcore/v1.8.0...sdk/azcore/v1.9.1) --- updated-dependencies: - dependency-name: github.com/Azure/azure-sdk-for-go/sdk/azcore dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index a18fd1fde..4d7ea07fe 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/restic/restic require ( cloud.google.com/go/storage v1.34.0 - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 github.com/Backblaze/blazer v0.6.1 @@ -41,7 +41,7 @@ require ( cloud.google.com/go/compute v1.23.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.3 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/dustin/go-humanize v1.0.1 // indirect diff --git a/go.sum b/go.sum index 61ed82f8b..9e739ef59 100644 --- a/go.sum +++ b/go.sum @@ -9,12 +9,12 @@ cloud.google.com/go/iam v1.1.3 h1:18tKG7DzydKWUnLjonWcJO6wjSCAtzh4GcRKlH/Hrzc= cloud.google.com/go/iam v1.1.3/go.mod h1:3khUlaBXfPKKe7huYgEpDn6FtgRyMEqbkvBxrQyY5SE= cloud.google.com/go/storage v1.34.0 h1:9KHBBTbaHPsNxO043SFmH3pMojjZiW+BFl9H41L7xjk= cloud.google.com/go/storage v1.34.0/go.mod h1:Eji+S0CCQebjsiXxyIvPItC3BN3zWsdJjWfHfoLblgY= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 h1:9kDVnTz3vbfweTqAUmk/a/pH5pWFCHtvRpHYC0G/dcA= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0/go.mod h1:3Ug6Qzto9anB6mGlEdgYMDF5zHQ+wwhEaYR4s17PHMw= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.4.0 h1:TuEMD+E+1aTjjLICGQOW6vLe8UWES7kopac9mUXL56Y= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.4.0/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0 h1:gggzg0SUMs6SQbEw+3LoSsYf9YMjkupeAnHMX8O9mmY= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.0/go.mod h1:+6KLcKIVgxoBDMqMO/Nvy7bZ9a0nbU3I1DtFQK3YvB4= From 2c60dd97aeb390440487486f302aa808392a123e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jan 2024 01:33:20 +0000 Subject: [PATCH 04/31] build(deps): bump actions/setup-go from 4 to 5 Bumps [actions/setup-go](https://github.com/actions/setup-go) from 4 to 5. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-go dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 291fab0f0..45681c6c5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -62,7 +62,7 @@ jobs: steps: - name: Set up Go ${{ matrix.go }} - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} @@ -226,7 +226,7 @@ jobs: steps: - name: Set up Go ${{ env.latest_go }} - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ env.latest_go }} @@ -244,7 +244,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Go ${{ env.latest_go }} - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ env.latest_go }} From 3666eef76c362ae05b99b48ceb118ede37a0e96a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Jan 2024 11:03:52 +0000 Subject: [PATCH 05/31] build(deps): bump github.com/minio/minio-go/v7 from 7.0.63 to 7.0.66 Bumps [github.com/minio/minio-go/v7](https://github.com/minio/minio-go) from 7.0.63 to 7.0.66. - [Release notes](https://github.com/minio/minio-go/releases) - [Commits](https://github.com/minio/minio-go/compare/v7.0.63...v7.0.66) --- updated-dependencies: - dependency-name: github.com/minio/minio-go/v7 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index eee8a65c4..debb0b0e0 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/google/go-cmp v0.6.0 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/klauspost/compress v1.17.4 - github.com/minio/minio-go/v7 v7.0.63 + github.com/minio/minio-go/v7 v7.0.66 github.com/minio/sha256-simd v1.0.1 github.com/ncw/swift/v2 v2.0.2 github.com/pkg/errors v0.9.1 @@ -51,12 +51,12 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect github.com/google/s2a-go v0.1.7 // indirect - github.com/google/uuid v1.4.0 // indirect + github.com/google/uuid v1.5.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/kr/fs v0.1.0 // indirect github.com/kr/text v0.2.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect diff --git a/go.sum b/go.sum index 2e889374f..377b926e8 100644 --- a/go.sum +++ b/go.sum @@ -95,8 +95,8 @@ github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0Z github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= -github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= @@ -111,8 +111,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= -github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= +github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -122,8 +122,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ= -github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4= +github.com/minio/minio-go/v7 v7.0.66 h1:bnTOXOHjOqv/gcMuiVbN9o2ngRItvqE774dG9nq0Dzw= +github.com/minio/minio-go/v7 v7.0.66/go.mod h1:DHAgmyQEGdW3Cif0UooKOyrT3Vxs82zNdV6tkKhRtbs= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= From 0018bb7854b4dc605f334d92e833b436142e212c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 18:43:16 +0100 Subject: [PATCH 06/31] restic: cleanup node type determination os.ModeCharDevice is already included in os.ModeType --- internal/restic/node.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/restic/node.go b/internal/restic/node.go index edb49bfca..1b940b0d0 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -109,7 +109,7 @@ func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) { } func nodeTypeFromFileInfo(fi os.FileInfo) string { - switch fi.Mode() & (os.ModeType | os.ModeCharDevice) { + switch fi.Mode() & os.ModeType { case 0: return "file" case os.ModeDir: From 6b79834cc8f1d77102db7d2e6a8890da2998cd68 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 19:03:11 +0100 Subject: [PATCH 07/31] archiver: improve error message for irregular files Since Go 1.21, most reparse points are considered as irregular files. Depending on the underlying driver these can exhibit nearly arbitrary behavior. When encountering such a file, restic returned an indecipherable error message: `error: invalid node type ""`. Add the filepath to the error message and state that the file type is not supported. --- internal/archiver/archiver.go | 6 +++++- internal/restic/node.go | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index e2f22ebea..8f9c8d8e8 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -2,6 +2,7 @@ package archiver import ( "context" + "fmt" "os" "path" "runtime" @@ -183,7 +184,10 @@ func (arch *Archiver) nodeFromFileInfo(snPath, filename string, fi os.FileInfo) } // overwrite name to match that within the snapshot node.Name = path.Base(snPath) - return node, errors.WithStack(err) + if err != nil { + return node, fmt.Errorf("nodeFromFileInfo %v: %w", filename, err) + } + return node, err } // loadSubtree tries to load the subtree referenced by node. In case of an error, nil is returned. diff --git a/internal/restic/node.go b/internal/restic/node.go index 1b940b0d0..7edc41ce8 100644 --- a/internal/restic/node.go +++ b/internal/restic/node.go @@ -124,6 +124,8 @@ func nodeTypeFromFileInfo(fi os.FileInfo) string { return "fifo" case os.ModeSocket: return "socket" + case os.ModeIrregular: + return "irregular" } return "" @@ -622,7 +624,7 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error { case "fifo": case "socket": default: - return errors.Errorf("invalid node type %q", node.Type) + return errors.Errorf("unsupported file type %q", node.Type) } return node.fillExtendedAttributes(path) From 51419c51d3cae9137e428acac9fb346220701b81 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 19:08:24 +0100 Subject: [PATCH 08/31] archiver: Add filepath to error message if it is not included yet --- internal/archiver/archiver.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/internal/archiver/archiver.go b/internal/archiver/archiver.go index 8f9c8d8e8..f2c481b32 100644 --- a/internal/archiver/archiver.go +++ b/internal/archiver/archiver.go @@ -7,6 +7,7 @@ import ( "path" "runtime" "sort" + "strings" "time" "github.com/restic/restic/internal/debug" @@ -169,6 +170,11 @@ func (arch *Archiver) error(item string, err error) error { return err } + // not all errors include the filepath, thus add it if it is missing + if !strings.Contains(err.Error(), item) { + err = fmt.Errorf("%v: %w", item, err) + } + errf := arch.Error(item, err) if err != errf { debug.Log("item %v: error was filtered by handler, before: %q, after: %v", item, err, errf) From a7dc18e6977018d74d158d7a96127d335124f5db Mon Sep 17 00:00:00 2001 From: Daniel Danner Date: Fri, 20 Oct 2023 20:15:53 +0200 Subject: [PATCH 09/31] Add bitrot detection to "diff" command This introduces a new modifier to the output of the diff command. It appears whenever two files being compared only differ in their content but not in their metadata. As far as we know, under normal circumstances, this should only ever happen if some kind of bitrot has happened in the source file. The prerequisite for this detection to work is that the right-side snapshot of the comparison has been created with "backup --force". --- changelog/unreleased/pull-4526 | 11 +++++++++++ cmd/restic/cmd_diff.go | 9 +++++++++ doc/075_scripting.rst | 3 ++- 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 changelog/unreleased/pull-4526 diff --git a/changelog/unreleased/pull-4526 b/changelog/unreleased/pull-4526 new file mode 100644 index 000000000..3a538f57a --- /dev/null +++ b/changelog/unreleased/pull-4526 @@ -0,0 +1,11 @@ +Enhancement: Add bitrot detection to `diff` command + +The output of the `diff` command now includes the modifier `?` for files +to indicate bitrot in backed up files. It will appear whenever there is a +difference in content while the metadata is exactly the same. Since files with +unchanged metadata are normally not read again when creating a backup, the +detection is only effective if the right-hand side of the diff has been created +with "backup --force". + +https://github.com/restic/restic/issues/805 +https://github.com/restic/restic/pull/4526 diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index c54fc06d4..329d11d02 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -27,6 +27,7 @@ directory: * U The metadata (access mode, timestamps, ...) for the item was updated * M The file's content was modified * T The type was changed, e.g. a file was made a symlink +* ? Bitrot detected: The file's content has changed but all metadata is the same To only compare files in specific subfolders, you can use the ":" syntax, where "subfolder" is a path within the @@ -272,6 +273,14 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref !reflect.DeepEqual(node1.Content, node2.Content) { mod += "M" stats.ChangedFiles++ + + node1NilContent := node1 + node2NilContent := node2 + node1NilContent.Content = nil + node2NilContent.Content = nil + if node1NilContent.Equals(*node2NilContent) { + mod += "?" + } } else if c.opts.ShowMetadata && !node1.Equals(*node2) { mod += "U" } diff --git a/doc/075_scripting.rst b/doc/075_scripting.rst index d94074232..f46572209 100644 --- a/doc/075_scripting.rst +++ b/doc/075_scripting.rst @@ -201,7 +201,8 @@ change +------------------+--------------------------------------------------------------+ | ``modifier`` | Type of change, a concatenation of the following characters: | | | "+" = added, "-" = removed, "T" = entry type changed, | -| | "M" = file content changed, "U" = metadata changed | +| | "M" = file content changed, "U" = metadata changed, | +| | "?" = bitrot detected | +------------------+--------------------------------------------------------------+ statistics From 3549635243d77679871fa6dd54b1d17b9ac5cb7b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 20:07:46 +0100 Subject: [PATCH 10/31] diff: copy nodes before modifying them for bitrot detection --- cmd/restic/cmd_diff.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index 329d11d02..aafc558b8 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -274,11 +274,13 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref mod += "M" stats.ChangedFiles++ - node1NilContent := node1 - node2NilContent := node2 + node1NilContent := *node1 + node2NilContent := *node2 node1NilContent.Content = nil node2NilContent.Content = nil - if node1NilContent.Equals(*node2NilContent) { + // the bitrot detection may not work if `backup --ignore-inode` or `--ignore-ctime` were used + if node1NilContent.Equals(node2NilContent) { + // probable bitrot detected mod += "?" } } else if c.opts.ShowMetadata && !node1.Equals(*node2) { From 4f6b1bb6f6388678345088a509e98a6560aee7a5 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 20:11:02 +0100 Subject: [PATCH 11/31] diff: document limitations regarding metadata --- cmd/restic/cmd_diff.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/restic/cmd_diff.go b/cmd/restic/cmd_diff.go index aafc558b8..ea40d2860 100644 --- a/cmd/restic/cmd_diff.go +++ b/cmd/restic/cmd_diff.go @@ -29,6 +29,9 @@ directory: * T The type was changed, e.g. a file was made a symlink * ? Bitrot detected: The file's content has changed but all metadata is the same +Metadata comparison will likely not work if a backup was created using the +'--ignore-inode' or '--ignore-ctime' option. + To only compare files in specific subfolders, you can use the ":" syntax, where "subfolder" is a path within the snapshot. From 045aa6455805e98f4c3f0d6d4d9a5479fc82e0c7 Mon Sep 17 00:00:00 2001 From: Michael Manganiello Date: Sat, 21 Oct 2023 16:16:06 -0300 Subject: [PATCH 12/31] cmd/copy: Prefix hostname to snapshot display output This change better resembles the output generated by `Snapshot.String()`, which includes both username and hostname. Closes #4506 Before: ``` $ restic copy --from-repo /srv/restic-repo repository 3666882b opened (version 2, compression level auto) repository 0085c387 opened (version 2, compression level auto) created new cache in /home/mike/.cache/restic [0:00] 100.00% 1 / 1 index files loaded [0:00] 0 index files loaded snapshot 32b39a20 of [/home/mike/data] at 2023-10-21 16:01:13.979948154 -0300 -03) copy started, this may take a while... [0:00] 100.00% 1 / 1 packs copied snapshot 10331fdd saved ``` After: ``` $ restic copy --from-repo /srv/restic-repo repository 3666882b opened (version 2, compression level auto) repository 0085c387 opened (version 2, compression level auto) [0:00] 100.00% 1 / 1 index files loaded [0:00] 0 index files loaded snapshot 32b39a20 of [/home/mike/data] at 2023-10-21 16:01:13.979948154 -0300 -03 by mike@desktop) copy started, this may take a while... [0:00] 100.00% 1 / 1 packs copied snapshot a67bd1ee saved ``` --- cmd/restic/cmd_copy.go | 5 +++-- cmd/restic/cmd_ls.go | 2 +- cmd/restic/cmd_repair_snapshots.go | 2 +- cmd/restic/cmd_rewrite.go | 2 +- doc/045_working_with_repos.rst | 10 +++++----- doc/077_troubleshooting.rst | 2 +- 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 0cf96a092..11e3a555f 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -126,11 +126,12 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] if sn.Original != nil { srcOriginal = *sn.Original } + if originalSns, ok := dstSnapshotByOriginal[srcOriginal]; ok { isCopy := false for _, originalSn := range originalSns { if similarSnapshots(originalSn, sn) { - Verboseff("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) + Verboseff("\nsnapshot %s of %v at %s by %s@%s)\n", sn.ID().Str(), sn.Paths, sn.Time, sn.Username, sn.Hostname) Verboseff("skipping source snapshot %s, was already copied to snapshot %s\n", sn.ID().Str(), originalSn.ID().Str()) isCopy = true break @@ -140,7 +141,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] continue } } - Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) + Verbosef("\nsnapshot %s of %v at %s by %s@%s)\n", sn.ID().Str(), sn.Paths, sn.Time, sn.Username, sn.Hostname) Verbosef(" copy started, this may take a while...\n") if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, gopts.Quiet); err != nil { return err diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 5b3984eb2..504e6a0fa 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -207,7 +207,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri } } else { printSnapshot = func(sn *restic.Snapshot) { - Verbosef("snapshot %s of %v filtered by %v at %s):\n", sn.ID().Str(), sn.Paths, dirs, sn.Time) + Verbosef("snapshot %s of %v filtered by %v at %s by %s@%s):\n", sn.ID().Str(), sn.Paths, dirs, sn.Time, sn.Username, sn.Hostname) } printNode = func(path string, node *restic.Node) { Printf("%s\n", formatNode(path, node, lsOptions.ListLong, lsOptions.HumanReadable)) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index 786097132..82cb7eede 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -144,7 +144,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt changedCount := 0 for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) { - Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) + Verbosef("\nsnapshot %s of %v at %s by %s@%s)\n", sn.ID().Str(), sn.Paths, sn.Time, sn.Username, sn.Hostname) changed, err := filterAndReplaceSnapshot(ctx, repo, sn, func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index afd80aca1..d31d26685 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -290,7 +290,7 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a changedCount := 0 for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) { - Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time) + Verbosef("\nsnapshot %s of %v at %s by %s@%s)\n", sn.ID().Str(), sn.Paths, sn.Time, sn.Username, sn.Hostname) changed, err := rewriteSnapshot(ctx, repo, sn, opts) if err != nil { return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err) diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index 68c181fa2..2fe8d1d8d 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -94,11 +94,11 @@ example from a local to a remote repository, you can use the ``copy`` command: repository d6504c63 opened successfully, password is correct repository 3dd0878c opened successfully, password is correct - snapshot 410b18a2 of [/home/user/work] at 2020-06-09 23:15:57.305305 +0200 CEST) + snapshot 410b18a2 of [/home/user/work] at 2020-06-09 23:15:57.305305 +0200 CEST by user@kasimir) copy started, this may take a while... snapshot 7a746a07 saved - snapshot 4e5d5487 of [/home/user/work] at 2020-05-01 22:44:07.012113 +0200 CEST) + snapshot 4e5d5487 of [/home/user/work] at 2020-05-01 22:44:07.012113 +0200 CEST by user@kasimir) skipping snapshot 4e5d5487, was already copied to snapshot 50eb62b7 The example command copies all snapshots from the source repository @@ -193,18 +193,18 @@ the unwanted files from affected snapshots by rewriting them using the $ restic -r /srv/restic-repo rewrite --exclude secret-file repository c881945a opened (repository version 2) successfully, password is correct - snapshot 6160ddb2 of [/home/user/work] at 2022-06-12 16:01:28.406630608 +0200 CEST) + snapshot 6160ddb2 of [/home/user/work] at 2022-06-12 16:01:28.406630608 +0200 CEST by user@kasimir) excluding /home/user/work/secret-file saved new snapshot b6aee1ff - snapshot 4fbaf325 of [/home/user/work] at 2022-05-01 11:22:26.500093107 +0200 CEST) + snapshot 4fbaf325 of [/home/user/work] at 2022-05-01 11:22:26.500093107 +0200 CEST by user@kasimir) modified 1 snapshots $ restic -r /srv/restic-repo rewrite --exclude secret-file 6160ddb2 repository c881945a opened (repository version 2) successfully, password is correct - snapshot 6160ddb2 of [/home/user/work] at 2022-06-12 16:01:28.406630608 +0200 CEST) + snapshot 6160ddb2 of [/home/user/work] at 2022-06-12 16:01:28.406630608 +0200 CEST by user@kasimir) excluding /home/user/work/secret-file new snapshot saved as b6aee1ff diff --git a/doc/077_troubleshooting.rst b/doc/077_troubleshooting.rst index fe317acfc..baf0be65c 100644 --- a/doc/077_troubleshooting.rst +++ b/doc/077_troubleshooting.rst @@ -153,7 +153,7 @@ command will automatically remove the original, damaged snapshots. $ restic repair snapshots --forget - snapshot 6979421e of [/home/user/restic/restic] at 2022-11-02 20:59:18.617503315 +0100 CET) + snapshot 6979421e of [/home/user/restic/restic] at 2022-11-02 20:59:18.617503315 +0100 CET by user@host) file "/restic/internal/fuse/snapshots_dir.go": removed missing content file "/restic/internal/restorer/restorer_unix_test.go": removed missing content file "/restic/internal/walker/walker.go": removed missing content From 33b7c84a7a323519be4b1b5f7a2ca3393748ba44 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 20:19:17 +0100 Subject: [PATCH 13/31] deduplicate string formatting of snapshot metadata This removes the spurious ")" bracket at the end and normalizes the metadata format used by the `ls` command. --- cmd/restic/cmd_copy.go | 4 ++-- cmd/restic/cmd_ls.go | 2 +- cmd/restic/cmd_repair_snapshots.go | 2 +- cmd/restic/cmd_rewrite.go | 2 +- internal/restic/snapshot.go | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/restic/cmd_copy.go b/cmd/restic/cmd_copy.go index 11e3a555f..92922b42b 100644 --- a/cmd/restic/cmd_copy.go +++ b/cmd/restic/cmd_copy.go @@ -131,7 +131,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] isCopy := false for _, originalSn := range originalSns { if similarSnapshots(originalSn, sn) { - Verboseff("\nsnapshot %s of %v at %s by %s@%s)\n", sn.ID().Str(), sn.Paths, sn.Time, sn.Username, sn.Hostname) + Verboseff("\n%v\n", sn) Verboseff("skipping source snapshot %s, was already copied to snapshot %s\n", sn.ID().Str(), originalSn.ID().Str()) isCopy = true break @@ -141,7 +141,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args [] continue } } - Verbosef("\nsnapshot %s of %v at %s by %s@%s)\n", sn.ID().Str(), sn.Paths, sn.Time, sn.Username, sn.Hostname) + Verbosef("\n%v\n", sn) Verbosef(" copy started, this may take a while...\n") if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, gopts.Quiet); err != nil { return err diff --git a/cmd/restic/cmd_ls.go b/cmd/restic/cmd_ls.go index 504e6a0fa..83a03559d 100644 --- a/cmd/restic/cmd_ls.go +++ b/cmd/restic/cmd_ls.go @@ -207,7 +207,7 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri } } else { printSnapshot = func(sn *restic.Snapshot) { - Verbosef("snapshot %s of %v filtered by %v at %s by %s@%s):\n", sn.ID().Str(), sn.Paths, dirs, sn.Time, sn.Username, sn.Hostname) + Verbosef("%v filtered by %v:\n", sn, dirs) } printNode = func(path string, node *restic.Node) { Printf("%s\n", formatNode(path, node, lsOptions.ListLong, lsOptions.HumanReadable)) diff --git a/cmd/restic/cmd_repair_snapshots.go b/cmd/restic/cmd_repair_snapshots.go index 82cb7eede..19e457b1f 100644 --- a/cmd/restic/cmd_repair_snapshots.go +++ b/cmd/restic/cmd_repair_snapshots.go @@ -144,7 +144,7 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt changedCount := 0 for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) { - Verbosef("\nsnapshot %s of %v at %s by %s@%s)\n", sn.ID().Str(), sn.Paths, sn.Time, sn.Username, sn.Hostname) + Verbosef("\n%v\n", sn) changed, err := filterAndReplaceSnapshot(ctx, repo, sn, func(ctx context.Context, sn *restic.Snapshot) (restic.ID, error) { return rewriter.RewriteTree(ctx, repo, "/", *sn.Tree) diff --git a/cmd/restic/cmd_rewrite.go b/cmd/restic/cmd_rewrite.go index d31d26685..d55e6137b 100644 --- a/cmd/restic/cmd_rewrite.go +++ b/cmd/restic/cmd_rewrite.go @@ -290,7 +290,7 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a changedCount := 0 for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, args) { - Verbosef("\nsnapshot %s of %v at %s by %s@%s)\n", sn.ID().Str(), sn.Paths, sn.Time, sn.Username, sn.Hostname) + Verbosef("\n%v\n", sn) changed, err := rewriteSnapshot(ctx, repo, sn, opts) if err != nil { return errors.Fatalf("unable to rewrite snapshot ID %q: %v", sn.ID().Str(), err) diff --git a/internal/restic/snapshot.go b/internal/restic/snapshot.go index 13e795ec8..88171a646 100644 --- a/internal/restic/snapshot.go +++ b/internal/restic/snapshot.go @@ -96,7 +96,7 @@ func ForAllSnapshots(ctx context.Context, be Lister, loader LoaderUnpacked, excl } func (sn Snapshot) String() string { - return fmt.Sprintf("", + return fmt.Sprintf("snapshot %s of %v at %s by %s@%s", sn.id.Str(), sn.Paths, sn.Time, sn.Username, sn.Hostname) } From c7844530d81a1fd074293e180f4ca434dcab098b Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 20:25:24 +0100 Subject: [PATCH 14/31] update docs --- doc/045_working_with_repos.rst | 12 ++++++------ doc/077_troubleshooting.rst | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/045_working_with_repos.rst b/doc/045_working_with_repos.rst index 2fe8d1d8d..d74c9c240 100644 --- a/doc/045_working_with_repos.rst +++ b/doc/045_working_with_repos.rst @@ -94,11 +94,11 @@ example from a local to a remote repository, you can use the ``copy`` command: repository d6504c63 opened successfully, password is correct repository 3dd0878c opened successfully, password is correct - snapshot 410b18a2 of [/home/user/work] at 2020-06-09 23:15:57.305305 +0200 CEST by user@kasimir) + snapshot 410b18a2 of [/home/user/work] at 2020-06-09 23:15:57.305305 +0200 CEST by user@kasimir copy started, this may take a while... snapshot 7a746a07 saved - snapshot 4e5d5487 of [/home/user/work] at 2020-05-01 22:44:07.012113 +0200 CEST by user@kasimir) + snapshot 4e5d5487 of [/home/user/work] at 2020-05-01 22:44:07.012113 +0200 CEST by user@kasimir skipping snapshot 4e5d5487, was already copied to snapshot 50eb62b7 The example command copies all snapshots from the source repository @@ -193,18 +193,18 @@ the unwanted files from affected snapshots by rewriting them using the $ restic -r /srv/restic-repo rewrite --exclude secret-file repository c881945a opened (repository version 2) successfully, password is correct - snapshot 6160ddb2 of [/home/user/work] at 2022-06-12 16:01:28.406630608 +0200 CEST by user@kasimir) + snapshot 6160ddb2 of [/home/user/work] at 2022-06-12 16:01:28.406630608 +0200 CEST by user@kasimir excluding /home/user/work/secret-file saved new snapshot b6aee1ff - snapshot 4fbaf325 of [/home/user/work] at 2022-05-01 11:22:26.500093107 +0200 CEST by user@kasimir) + snapshot 4fbaf325 of [/home/user/work] at 2022-05-01 11:22:26.500093107 +0200 CEST by user@kasimir modified 1 snapshots $ restic -r /srv/restic-repo rewrite --exclude secret-file 6160ddb2 repository c881945a opened (repository version 2) successfully, password is correct - snapshot 6160ddb2 of [/home/user/work] at 2022-06-12 16:01:28.406630608 +0200 CEST by user@kasimir) + snapshot 6160ddb2 of [/home/user/work] at 2022-06-12 16:01:28.406630608 +0200 CEST by user@kasimir excluding /home/user/work/secret-file new snapshot saved as b6aee1ff @@ -247,7 +247,7 @@ This is possible using the ``rewrite`` command with the option ``--new-host`` fo repository b7dbade3 opened (version 2, compression level auto) [0:00] 100.00% 1 / 1 index files loaded - snapshot 8ed674f4 of [/path/to/abc.txt] at 2023-11-27 21:57:52.439139291 +0100 CET) + snapshot 8ed674f4 of [/path/to/abc.txt] at 2023-11-27 21:57:52.439139291 +0100 CET by user@kasimir setting time to 1999-01-01 11:11:11 +0100 CET setting host to newhost saved new snapshot c05da643 diff --git a/doc/077_troubleshooting.rst b/doc/077_troubleshooting.rst index baf0be65c..6a9a6ee15 100644 --- a/doc/077_troubleshooting.rst +++ b/doc/077_troubleshooting.rst @@ -153,7 +153,7 @@ command will automatically remove the original, damaged snapshots. $ restic repair snapshots --forget - snapshot 6979421e of [/home/user/restic/restic] at 2022-11-02 20:59:18.617503315 +0100 CET by user@host) + snapshot 6979421e of [/home/user/restic/restic] at 2022-11-02 20:59:18.617503315 +0100 CET by user@host file "/restic/internal/fuse/snapshots_dir.go": removed missing content file "/restic/internal/restorer/restorer_unix_test.go": removed missing content file "/restic/internal/walker/walker.go": removed missing content From 5ffb536aaed98e383b037563dd15509e71cf7620 Mon Sep 17 00:00:00 2001 From: Erik Kristensen Date: Mon, 18 Sep 2023 12:09:32 -0600 Subject: [PATCH 15/31] feat: support AWS assume role --- .gitignore | 1 + changelog/unreleased/issue-4472 | 14 ++++ doc/040_backup.rst | 4 ++ internal/backend/s3/s3.go | 122 +++++++++++++++++++++++--------- 4 files changed, 108 insertions(+), 33 deletions(-) create mode 100644 changelog/unreleased/issue-4472 diff --git a/.gitignore b/.gitignore index b7201c26b..c8c3aa69a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +/.idea /restic /restic.exe /.vagrant diff --git a/changelog/unreleased/issue-4472 b/changelog/unreleased/issue-4472 new file mode 100644 index 000000000..97553f946 --- /dev/null +++ b/changelog/unreleased/issue-4472 @@ -0,0 +1,14 @@ +Enhancement: Allow AWS Assume Role to be used for S3 backend + +Previously only credentials discovered via the Minio Click discovery methods +would be used to authenticate. However there are many circumstances where the +discovered credentials have lower permissions and need to assume a specific role. + +New Environment Variables: + +- RESTIC_AWS_ASSUME_ROLE_ARN +- RESTIC_AWS_ASSUME_ROLE_SESSION_NAME +- RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID +- RESTIC_AWS_ASSUME_ROLE_REGION (if need to override from us-east-1) +- RESTIC_AWS_ASSUME_ROLE_POLICY +- RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT \ No newline at end of file diff --git a/doc/040_backup.rst b/doc/040_backup.rst index acafe2694..3de8ef554 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -628,6 +628,10 @@ environment variables. The following lists these environment variables: AWS_DEFAULT_REGION Amazon S3 default region AWS_PROFILE Amazon credentials profile (alternative to specifying key and region) AWS_SHARED_CREDENTIALS_FILE Location of the AWS CLI shared credentials file (default: ~/.aws/credentials) + RESTIC_AWS_ASSUME_ROLE_ARN Amazon IAM Role ARN to assume using discovered credentials + RESTIC_AWS_ASSUME_ROLE_SESSION_NAME Session Name to use with the role assumption + RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID External ID to use with the role assumption + RESTIC_AWS_ASSUME_ROLE_REGION Region to use for IAM calls for the role assumption AZURE_ACCOUNT_NAME Account name for Azure AZURE_ACCOUNT_KEY Account key for Azure diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index 98879d0df..ff81a05d6 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -51,40 +51,9 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro minio.MaxRetry = int(cfg.MaxRetries) } - // Chains all credential types, in the following order: - // - Static credentials provided by user - // - AWS env vars (i.e. AWS_ACCESS_KEY_ID) - // - Minio env vars (i.e. MINIO_ACCESS_KEY) - // - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials) - // - Minio creds file (i.e. MINIO_SHARED_CREDENTIALS_FILE or ~/.mc/config.json) - // - IAM profile based credentials. (performs an HTTP - // call to a pre-defined endpoint, only valid inside - // configured ec2 instances) - creds := credentials.NewChainCredentials([]credentials.Provider{ - &credentials.EnvAWS{}, - &credentials.Static{ - Value: credentials.Value{ - AccessKeyID: cfg.KeyID, - SecretAccessKey: cfg.Secret.Unwrap(), - }, - }, - &credentials.EnvMinio{}, - &credentials.FileAWSCredentials{}, - &credentials.FileMinioClient{}, - &credentials.IAM{ - Client: &http.Client{ - Transport: http.DefaultTransport, - }, - }, - }) - - c, err := creds.Get() + creds, err := getCredentials(cfg) if err != nil { - return nil, errors.Wrap(err, "creds.Get") - } - - if c.SignerType == credentials.SignatureAnonymous { - debug.Log("using anonymous access for %#v", cfg.Endpoint) + return nil, errors.Wrap(err, "s3.getCredentials") } options := &minio.Options{ @@ -125,6 +94,93 @@ func open(ctx context.Context, cfg Config, rt http.RoundTripper) (*Backend, erro return be, nil } +// getCredentials -- runs through the various credential types and returns the first one that works. +// additionally if the user has specified a role to assume, it will do that as well. +func getCredentials(cfg Config) (*credentials.Credentials, error) { + // Chains all credential types, in the following order: + // - Static credentials provided by user + // - AWS env vars (i.e. AWS_ACCESS_KEY_ID) + // - Minio env vars (i.e. MINIO_ACCESS_KEY) + // - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials) + // - Minio creds file (i.e. MINIO_SHARED_CREDENTIALS_FILE or ~/.mc/config.json) + // - IAM profile based credentials. (performs an HTTP + // call to a pre-defined endpoint, only valid inside + // configured ec2 instances) + creds := credentials.NewChainCredentials([]credentials.Provider{ + &credentials.Static{ + Value: credentials.Value{ + AccessKeyID: cfg.KeyID, + SecretAccessKey: cfg.Secret.Unwrap(), + }, + }, + &credentials.EnvAWS{}, + &credentials.EnvMinio{}, + &credentials.FileAWSCredentials{}, + &credentials.FileMinioClient{}, + &credentials.IAM{ + Client: &http.Client{ + Transport: http.DefaultTransport, + }, + }, + }) + + c, err := creds.Get() + if err != nil { + return nil, errors.Wrap(err, "creds.Get") + } + + if c.SignerType == credentials.SignatureAnonymous { + debug.Log("using anonymous access for %#v", cfg.Endpoint) + } + + roleArn := os.Getenv("RESTIC_AWS_ASSUME_ROLE_ARN") + if roleArn != "" { + // use the region provided by the configuration by default + awsRegion := cfg.Region + // allow the region to be overridden if for some reason it is required + if len(os.Getenv("RESTIC_AWS_ASSUME_ROLE_REGION")) > 0 { + awsRegion = os.Getenv("RESTIC_AWS_ASSUME_ROLE_REGION") + } + + sessionName := os.Getenv("RESTIC_AWS_ASSUME_ROLE_SESSION_NAME") + externalID := os.Getenv("RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID") + policy := os.Getenv("RESTIC_AWS_ASSUME_ROLE_POLICY") + stsEndpoint := os.Getenv("RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT") + + if stsEndpoint == "" { + if len(awsRegion) > 0 { + if strings.HasPrefix(awsRegion, "cn-") { + stsEndpoint = "https://sts." + awsRegion + ".amazonaws.com.cn" + } else { + stsEndpoint = "https://sts." + awsRegion + ".amazonaws.com" + } + } else { + stsEndpoint = "https://sts.amazonaws.com" + } + } + + opts := credentials.STSAssumeRoleOptions{ + RoleARN: roleArn, + AccessKey: c.AccessKeyID, + SecretKey: c.SecretAccessKey, + SessionToken: c.SessionToken, + RoleSessionName: sessionName, + ExternalID: externalID, + Policy: policy, + } + if len(awsRegion) > 0 { + opts.Location = awsRegion + } + + creds, err = credentials.NewSTSAssumeRole(stsEndpoint, opts) + if err != nil { + return nil, errors.Wrap(err, "creds.AssumeRole") + } + } + + return creds, nil +} + // Open opens the S3 backend at bucket and region. The bucket is created if it // does not exist yet. func Open(ctx context.Context, cfg Config, rt http.RoundTripper) (backend.Backend, error) { From 20cf4777cbf52b4b76777faaf20ceb4ba5a71df5 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 21:43:41 +0100 Subject: [PATCH 16/31] s3: check for EnvAWS credentials before Static credentials EnvAWS considers more environment variables, including AWS_SESSION_TOKEN and thus should be checked first. --- internal/backend/s3/s3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index ff81a05d6..d48813cf2 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -107,13 +107,13 @@ func getCredentials(cfg Config) (*credentials.Credentials, error) { // call to a pre-defined endpoint, only valid inside // configured ec2 instances) creds := credentials.NewChainCredentials([]credentials.Provider{ + &credentials.EnvAWS{}, &credentials.Static{ Value: credentials.Value{ AccessKeyID: cfg.KeyID, SecretAccessKey: cfg.Secret.Unwrap(), }, }, - &credentials.EnvAWS{}, &credentials.EnvMinio{}, &credentials.FileAWSCredentials{}, &credentials.FileMinioClient{}, From 02bc73f5eb2ed392aaffa510fbe1cfd392711d28 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 21:44:53 +0100 Subject: [PATCH 17/31] s3: minor code cleanups --- internal/backend/s3/s3.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/internal/backend/s3/s3.go b/internal/backend/s3/s3.go index d48813cf2..f0447224f 100644 --- a/internal/backend/s3/s3.go +++ b/internal/backend/s3/s3.go @@ -138,7 +138,7 @@ func getCredentials(cfg Config) (*credentials.Credentials, error) { // use the region provided by the configuration by default awsRegion := cfg.Region // allow the region to be overridden if for some reason it is required - if len(os.Getenv("RESTIC_AWS_ASSUME_ROLE_REGION")) > 0 { + if os.Getenv("RESTIC_AWS_ASSUME_ROLE_REGION") != "" { awsRegion = os.Getenv("RESTIC_AWS_ASSUME_ROLE_REGION") } @@ -148,7 +148,7 @@ func getCredentials(cfg Config) (*credentials.Credentials, error) { stsEndpoint := os.Getenv("RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT") if stsEndpoint == "" { - if len(awsRegion) > 0 { + if awsRegion != "" { if strings.HasPrefix(awsRegion, "cn-") { stsEndpoint = "https://sts." + awsRegion + ".amazonaws.com.cn" } else { @@ -167,9 +167,7 @@ func getCredentials(cfg Config) (*credentials.Credentials, error) { RoleSessionName: sessionName, ExternalID: externalID, Policy: policy, - } - if len(awsRegion) > 0 { - opts.Location = awsRegion + Location: awsRegion, } creds, err = credentials.NewSTSAssumeRole(stsEndpoint, opts) From e6dfefba139c03000d38d43dd910a72e99611c44 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 21:59:26 +0100 Subject: [PATCH 18/31] termstatus: update import path of golang.org/x/term --- internal/ui/termstatus/terminal_windows.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/ui/termstatus/terminal_windows.go b/internal/ui/termstatus/terminal_windows.go index 7bf5b0a37..3603f16a3 100644 --- a/internal/ui/termstatus/terminal_windows.go +++ b/internal/ui/termstatus/terminal_windows.go @@ -9,8 +9,8 @@ import ( "syscall" "unsafe" - "golang.org/x/crypto/ssh/terminal" "golang.org/x/sys/windows" + "golang.org/x/term" ) // clearCurrentLine removes all characters from the current line and resets the @@ -74,7 +74,7 @@ func windowsMoveCursorUp(_ io.Writer, fd uintptr, n int) { // isWindowsTerminal return true if the file descriptor is a windows terminal (cmd, psh). func isWindowsTerminal(fd uintptr) bool { - return terminal.IsTerminal(int(fd)) + return term.IsTerminal(int(fd)) } func isPipe(fd uintptr) bool { From 6ef23b401b53bfbfd7d3c68e8137a0d7584ca53c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 19:32:38 +0100 Subject: [PATCH 19/31] fix deduplicated files on windows --- changelog/unreleased/issue-4574 | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 changelog/unreleased/issue-4574 diff --git a/changelog/unreleased/issue-4574 b/changelog/unreleased/issue-4574 new file mode 100644 index 000000000..3668ae6c3 --- /dev/null +++ b/changelog/unreleased/issue-4574 @@ -0,0 +1,11 @@ +Bugfix: support backup of deduplicated files on Windows again + +With the official release builds of restic 0.16.1 and 0.16.2, it was not +possible to back up files that were deduplicated by the corresponding Windows +Server feature. This also applies to restic versions built using Go +1.21.0 - 1.21.4. + +We have updated the used Go version to fix this. + +https://github.com/restic/restic/issues/4574 +https://github.com/restic/restic/pull/4621 From 3e29f8dddfb5a568c2feffad99cf8c840835acf5 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 19:17:54 +0100 Subject: [PATCH 20/31] add changelog for irregular files on windows --- changelog/unreleased/issue-4560 | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 changelog/unreleased/issue-4560 diff --git a/changelog/unreleased/issue-4560 b/changelog/unreleased/issue-4560 new file mode 100644 index 000000000..c421f6e69 --- /dev/null +++ b/changelog/unreleased/issue-4560 @@ -0,0 +1,14 @@ +Bugfix: Improve errors for irregular files on Windows + +Since Go 1.21, most filesystem reparse points on Windows are considered to be +irregular files. This caused restic to show an `error: invalid node type ""` +error message for those files. + +We have improved the error message to include the file path for those files: +`error: nodeFromFileInfo path/to/file: unsupported file type "irregular"`. +As irregular files are not required to behave like regular files, it is not +possible to provide a generic way to back up those files. + +https://github.com/restic/restic/issues/4560 +https://github.com/restic/restic/pull/4620 +https://forum.restic.net/t/windows-backup-error-invalid-node-type/6875 From fe5c337ca259742c956048d4abe05fa99a8b15fa Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 30 Dec 2023 21:40:41 +0100 Subject: [PATCH 21/31] repository: StreamPack delivers blobs at most once If an error occurred while streaming a pack file, this could result in passing some of the blobs multiple times to the callback function. This significantly complicates using StreamPack correctly and is unnecessary. Retries do not change the content of a blob and thus only deliver the same result over and over again. --- internal/repository/repository.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/repository/repository.go b/internal/repository/repository.go index f78c55e1d..97dc33fdf 100644 --- a/internal/repository/repository.go +++ b/internal/repository/repository.go @@ -881,9 +881,9 @@ type BackendLoadFn func(ctx context.Context, h backend.Handle, length int, offse const maxUnusedRange = 4 * 1024 * 1024 // StreamPack loads the listed blobs from the specified pack file. The plaintext blob is passed to -// the handleBlobFn callback or an error if decryption failed or the blob hash does not match. In -// case of download errors handleBlobFn might be called multiple times for the same blob. If the -// callback returns an error, then StreamPack will abort and not retry it. +// the handleBlobFn callback or an error if decryption failed or the blob hash does not match. +// handleBlobFn is never called multiple times for the same blob. If the callback returns an error, +// then StreamPack will abort and not retry it. func StreamPack(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, packID restic.ID, blobs []restic.Blob, handleBlobFn func(blob restic.BlobHandle, buf []byte, err error) error) error { if len(blobs) == 0 { // nothing to do @@ -945,7 +945,9 @@ func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, currentBlobEnd := dataStart var buf []byte var decode []byte - for _, entry := range blobs { + for len(blobs) > 0 { + entry := blobs[0] + skipBytes := int(entry.Offset - currentBlobEnd) if skipBytes < 0 { return errors.Errorf("overlapping blobs in pack %v", packID) @@ -1008,6 +1010,8 @@ func streamPackPart(ctx context.Context, beLoad BackendLoadFn, key *crypto.Key, cancel() return backoff.Permanent(err) } + // ensure that each blob is only passed once to handleBlobFn + blobs = blobs[1:] } return nil }) From 77b1c52673a779929058d045ab87e6949f6b6478 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 30 Dec 2023 22:08:57 +0100 Subject: [PATCH 22/31] repository: test that StreamPack only delivers blobs once --- internal/repository/repository_test.go | 50 ++++++++++++++++++++------ 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/internal/repository/repository_test.go b/internal/repository/repository_test.go index bb8395436..1178a7693 100644 --- a/internal/repository/repository_test.go +++ b/internal/repository/repository_test.go @@ -5,6 +5,7 @@ import ( "context" "crypto/sha256" "encoding/json" + "errors" "fmt" "io" "math/rand" @@ -14,6 +15,7 @@ import ( "testing" "time" + "github.com/cenkalti/backoff/v4" "github.com/google/go-cmp/cmp" "github.com/klauspost/compress/zstd" "github.com/restic/restic/internal/backend" @@ -529,7 +531,9 @@ func testStreamPack(t *testing.T, version uint) { packfileBlobs, packfile := buildPackfileWithoutHeader(blobSizes, &key, compress) loadCalls := 0 - load := func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + shortFirstLoad := false + + loadBytes := func(length int, offset int64) []byte { data := packfile if offset > int64(len(data)) { @@ -541,32 +545,56 @@ func testStreamPack(t *testing.T, version uint) { if length > len(data) { length = len(data) } + if shortFirstLoad { + length /= 2 + shortFirstLoad = false + } + + return data[:length] + } + + load := func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + data := loadBytes(length, offset) + if shortFirstLoad { + data = data[:len(data)/2] + shortFirstLoad = false + } - data = data[:length] loadCalls++ - return fn(bytes.NewReader(data)) + err := fn(bytes.NewReader(data)) + if err == nil { + return nil + } + var permanent *backoff.PermanentError + if errors.As(err, &permanent) { + return err + } + // retry loading once + return fn(bytes.NewReader(loadBytes(length, offset))) } // first, test regular usage t.Run("regular", func(t *testing.T) { tests := []struct { - blobs []restic.Blob - calls int + blobs []restic.Blob + calls int + shortFirstLoad bool }{ - {packfileBlobs[1:2], 1}, - {packfileBlobs[2:5], 1}, - {packfileBlobs[2:8], 1}, + {packfileBlobs[1:2], 1, false}, + {packfileBlobs[2:5], 1, false}, + {packfileBlobs[2:8], 1, false}, {[]restic.Blob{ packfileBlobs[0], packfileBlobs[4], packfileBlobs[2], - }, 1}, + }, 1, false}, {[]restic.Blob{ packfileBlobs[0], packfileBlobs[len(packfileBlobs)-1], - }, 2}, + }, 2, false}, + {packfileBlobs[:], 1, true}, } for _, test := range tests { @@ -593,6 +621,7 @@ func testStreamPack(t *testing.T, version uint) { } loadCalls = 0 + shortFirstLoad = test.shortFirstLoad err = repository.StreamPack(ctx, load, &key, restic.ID{}, test.blobs, handleBlob) if err != nil { t.Fatal(err) @@ -605,6 +634,7 @@ func testStreamPack(t *testing.T, version uint) { }) } }) + shortFirstLoad = false // next, test invalid uses, which should return an error t.Run("invalid", func(t *testing.T) { From dac350817081b59e72325834161f880fa46576cb Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 30 Dec 2023 22:39:26 +0100 Subject: [PATCH 23/31] restore: only report errors for blobs that actually failed to load Previously, errors would be reported for all blobs of a packfile that failed to stream. Now, only the not yet processed blobs are reported. --- internal/restorer/filerestorer.go | 17 +++++++++- internal/restorer/filerestorer_test.go | 44 ++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index 3bb7489ba..1fc74c7f0 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -246,7 +246,10 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { return err } + // track already processed blobs for precise error reporting + processedBlobs := restic.NewBlobSet() err := repository.StreamPack(ctx, r.packLoader, r.key, pack.id, blobList, func(h restic.BlobHandle, blobData []byte, err error) error { + processedBlobs.Insert(h) blob := blobs[h.ID] if err != nil { for file := range blob.files { @@ -292,7 +295,19 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { }) if err != nil { - for file := range pack.files { + // only report error for not yet processed blobs + affectedFiles := make(map[*fileInfo]struct{}) + for _, blob := range blobList { + if processedBlobs.Has(blob.BlobHandle) { + continue + } + blob := blobs[blob.ID] + for file := range blob.files { + affectedFiles[file] = struct{}{} + } + } + + for file := range affectedFiles { if errFile := sanitizeError(file, err); errFile != nil { return errFile } diff --git a/internal/restorer/filerestorer_test.go b/internal/restorer/filerestorer_test.go index aa9a2392d..94b068159 100644 --- a/internal/restorer/filerestorer_test.go +++ b/internal/restorer/filerestorer_test.go @@ -317,3 +317,47 @@ func testPartialDownloadError(t *testing.T, part int) { rtest.OK(t, err) verifyRestore(t, r, repo) } + +func TestFatalDownloadError(t *testing.T) { + tempdir := rtest.TempDir(t) + content := []TestFile{ + { + name: "file1", + blobs: []TestBlob{ + {"data1-1", "pack1"}, + {"data1-2", "pack1"}, + }, + }, + { + name: "file2", + blobs: []TestBlob{ + {"data2-1", "pack1"}, + {"data2-2", "pack1"}, + {"data2-3", "pack1"}, + }, + }} + + repo := newTestRepo(content) + + loader := repo.loader + repo.loader = func(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error { + // only return half the data to break file2 + return loader(ctx, h, length/2, offset, fn) + } + + r := newFileRestorer(tempdir, repo.loader, repo.key, repo.Lookup, 2, false, nil) + r.files = repo.files + + var errors []string + r.Error = func(s string, e error) error { + // ignore errors as in the `restore` command + errors = append(errors, s) + return nil + } + + err := r.restoreFiles(context.TODO()) + rtest.OK(t, err) + + rtest.Assert(t, len(errors) == 1, "unexpected number of restore errors, expected: 1, got: %v", len(errors)) + rtest.Assert(t, errors[0] == "file2", "expected error for file2, got: %v", errors[0]) +} From 100872308f938d285fc4759c47b713d8e189d14f Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 7 Jan 2024 11:06:42 +0100 Subject: [PATCH 24/31] add changelog for better restore error reporting --- changelog/unreleased/pull-4624 | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 changelog/unreleased/pull-4624 diff --git a/changelog/unreleased/pull-4624 b/changelog/unreleased/pull-4624 new file mode 100644 index 000000000..fbdbb1558 --- /dev/null +++ b/changelog/unreleased/pull-4624 @@ -0,0 +1,11 @@ +Bugfix: Correct restore progress information if an error occurs + +If an error occurred while restoring a snapshot, this could cause the restore +progress bar to show incorrect information. In addition, if a data file could +not be loaded completely, then errors would also be reported for some already +restored files. + +We have improved the error reporting of the restore command to be more accurate. + +https://github.com/restic/restic/pull/4624 +https://forum.restic.net/t/errors-restoring-with-restic-on-windows-server-s3/6943 From 4248c6c3cab07b74a59f7813db75d2d52a59ca6a Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 21:45:09 +0100 Subject: [PATCH 25/31] s3: update documentation --- changelog/unreleased/issue-4472 | 16 ++++++++++------ doc/040_backup.rst | 4 +++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/changelog/unreleased/issue-4472 b/changelog/unreleased/issue-4472 index 97553f946..3049fdf30 100644 --- a/changelog/unreleased/issue-4472 +++ b/changelog/unreleased/issue-4472 @@ -1,14 +1,18 @@ Enhancement: Allow AWS Assume Role to be used for S3 backend -Previously only credentials discovered via the Minio Click discovery methods -would be used to authenticate. However there are many circumstances where the -discovered credentials have lower permissions and need to assume a specific role. +Previously only credentials discovered via the Minio discovery methods +were used to authenticate. -New Environment Variables: +However, there are many circumstances where the discovered credentials have +lower permissions and need to assume a specific role. This is now possible +using the following new environment variables. - RESTIC_AWS_ASSUME_ROLE_ARN - RESTIC_AWS_ASSUME_ROLE_SESSION_NAME - RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID -- RESTIC_AWS_ASSUME_ROLE_REGION (if need to override from us-east-1) +- RESTIC_AWS_ASSUME_ROLE_REGION (defaults to us-east-1) - RESTIC_AWS_ASSUME_ROLE_POLICY -- RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT \ No newline at end of file +- RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT + +https://github.com/restic/restic/issues/4472 +https://github.com/restic/restic/pull/4474 diff --git a/doc/040_backup.rst b/doc/040_backup.rst index 3de8ef554..d36986441 100644 --- a/doc/040_backup.rst +++ b/doc/040_backup.rst @@ -631,7 +631,9 @@ environment variables. The following lists these environment variables: RESTIC_AWS_ASSUME_ROLE_ARN Amazon IAM Role ARN to assume using discovered credentials RESTIC_AWS_ASSUME_ROLE_SESSION_NAME Session Name to use with the role assumption RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID External ID to use with the role assumption - RESTIC_AWS_ASSUME_ROLE_REGION Region to use for IAM calls for the role assumption + RESTIC_AWS_ASSUME_ROLE_POLICY Inline Amazion IAM session policy + RESTIC_AWS_ASSUME_ROLE_REGION Region to use for IAM calls for the role assumption (default: us-east-1) + RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT URL to the STS endpoint (default is determined based on RESTIC_AWS_ASSUME_ROLE_REGION). You generally do not need to set this, advanced use only. AZURE_ACCOUNT_NAME Account name for Azure AZURE_ACCOUNT_KEY Account key for Azure From 9328f34d4363832344f450cbaa8e01d1cb135114 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 7 Jan 2024 12:00:32 +0100 Subject: [PATCH 26/31] restore: split downloadPack into smaller methods --- internal/restorer/filerestorer.go | 122 ++++++++++++++++-------------- 1 file changed, 65 insertions(+), 57 deletions(-) diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index 1fc74c7f0..7621e5ebb 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -197,12 +197,13 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { return wg.Wait() } -func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { +type blobToFileOffsetsMapping map[restic.ID]struct { + files map[*fileInfo][]int64 // file -> offsets (plural!) of the blob in the file +} +func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { // calculate blob->[]files->[]offsets mappings - blobs := make(map[restic.ID]struct { - files map[*fileInfo][]int64 // file -> offsets (plural!) of the blob in the file - }) + blobs := make(blobToFileOffsetsMapping) var blobList []restic.Blob for file := range pack.files { addBlob := func(blob restic.Blob, fileOffset int64) { @@ -239,60 +240,9 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { } } - sanitizeError := func(file *fileInfo, err error) error { - if err != nil { - err = r.Error(file.location, err) - } - return err - } - // track already processed blobs for precise error reporting processedBlobs := restic.NewBlobSet() - err := repository.StreamPack(ctx, r.packLoader, r.key, pack.id, blobList, func(h restic.BlobHandle, blobData []byte, err error) error { - processedBlobs.Insert(h) - blob := blobs[h.ID] - if err != nil { - for file := range blob.files { - if errFile := sanitizeError(file, err); errFile != nil { - return errFile - } - } - return nil - } - for file, offsets := range blob.files { - for _, offset := range offsets { - writeToFile := func() error { - // this looks overly complicated and needs explanation - // two competing requirements: - // - must create the file once and only once - // - should allow concurrent writes to the file - // so write the first blob while holding file lock - // write other blobs after releasing the lock - createSize := int64(-1) - file.lock.Lock() - if file.inProgress { - file.lock.Unlock() - } else { - defer file.lock.Unlock() - file.inProgress = true - createSize = file.size - } - writeErr := r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize, file.sparse) - - if r.progress != nil { - r.progress.AddProgress(file.location, uint64(len(blobData)), uint64(file.size)) - } - - return writeErr - } - err := sanitizeError(file, writeToFile()) - if err != nil { - return err - } - } - } - return nil - }) + err := r.downloadBlobs(ctx, pack.id, blobList, blobs, processedBlobs) if err != nil { // only report error for not yet processed blobs @@ -308,7 +258,7 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { } for file := range affectedFiles { - if errFile := sanitizeError(file, err); errFile != nil { + if errFile := r.sanitizeError(file, err); errFile != nil { return errFile } } @@ -316,3 +266,61 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { return nil } + +func (r *fileRestorer) sanitizeError(file *fileInfo, err error) error { + if err != nil { + err = r.Error(file.location, err) + } + return err +} + +func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID, blobList []restic.Blob, + blobs blobToFileOffsetsMapping, processedBlobs restic.BlobSet) error { + + return repository.StreamPack(ctx, r.packLoader, r.key, packID, blobList, + func(h restic.BlobHandle, blobData []byte, err error) error { + processedBlobs.Insert(h) + blob := blobs[h.ID] + if err != nil { + for file := range blob.files { + if errFile := r.sanitizeError(file, err); errFile != nil { + return errFile + } + } + return nil + } + for file, offsets := range blob.files { + for _, offset := range offsets { + writeToFile := func() error { + // this looks overly complicated and needs explanation + // two competing requirements: + // - must create the file once and only once + // - should allow concurrent writes to the file + // so write the first blob while holding file lock + // write other blobs after releasing the lock + createSize := int64(-1) + file.lock.Lock() + if file.inProgress { + file.lock.Unlock() + } else { + defer file.lock.Unlock() + file.inProgress = true + createSize = file.size + } + writeErr := r.filesWriter.writeToFile(r.targetPath(file.location), blobData, offset, createSize, file.sparse) + + if r.progress != nil { + r.progress.AddProgress(file.location, uint64(len(blobData)), uint64(file.size)) + } + + return writeErr + } + err := r.sanitizeError(file, writeToFile()) + if err != nil { + return err + } + } + } + return nil + }) +} From 00d18b7a8847796d709d7de6e7bce3daeaf493b5 Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 7 Jan 2024 12:06:36 +0100 Subject: [PATCH 27/31] restore: cleanup downloadPack --- internal/restorer/filerestorer.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index 7621e5ebb..403651763 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -199,18 +199,18 @@ func (r *fileRestorer) restoreFiles(ctx context.Context) error { type blobToFileOffsetsMapping map[restic.ID]struct { files map[*fileInfo][]int64 // file -> offsets (plural!) of the blob in the file + blob restic.Blob } func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { // calculate blob->[]files->[]offsets mappings blobs := make(blobToFileOffsetsMapping) - var blobList []restic.Blob for file := range pack.files { addBlob := func(blob restic.Blob, fileOffset int64) { blobInfo, ok := blobs[blob.ID] if !ok { blobInfo.files = make(map[*fileInfo][]int64) - blobList = append(blobList, blob) + blobInfo.blob = blob blobs[blob.ID] = blobInfo } blobInfo.files[file] = append(blobInfo.files[file], fileOffset) @@ -242,17 +242,16 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { // track already processed blobs for precise error reporting processedBlobs := restic.NewBlobSet() - err := r.downloadBlobs(ctx, pack.id, blobList, blobs, processedBlobs) + err := r.downloadBlobs(ctx, pack.id, blobs, processedBlobs) if err != nil { // only report error for not yet processed blobs affectedFiles := make(map[*fileInfo]struct{}) - for _, blob := range blobList { - if processedBlobs.Has(blob.BlobHandle) { + for _, entry := range blobs { + if processedBlobs.Has(entry.blob.BlobHandle) { continue } - blob := blobs[blob.ID] - for file := range blob.files { + for file := range entry.files { affectedFiles[file] = struct{}{} } } @@ -274,9 +273,13 @@ func (r *fileRestorer) sanitizeError(file *fileInfo, err error) error { return err } -func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID, blobList []restic.Blob, +func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID, blobs blobToFileOffsetsMapping, processedBlobs restic.BlobSet) error { + blobList := make([]restic.Blob, 0, len(blobs)) + for _, entry := range blobs { + blobList = append(blobList, entry.blob) + } return repository.StreamPack(ctx, r.packLoader, r.key, packID, blobList, func(h restic.BlobHandle, blobData []byte, err error) error { processedBlobs.Insert(h) From 226791041852a522e71d1145f3361658609b05fe Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 7 Jan 2024 12:20:31 +0100 Subject: [PATCH 28/31] restore: split error reporting from downloadPack --- internal/restorer/filerestorer.go | 45 +++++++++++++++++-------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index 403651763..f2e2cf24a 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -244,26 +244,7 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { processedBlobs := restic.NewBlobSet() err := r.downloadBlobs(ctx, pack.id, blobs, processedBlobs) - if err != nil { - // only report error for not yet processed blobs - affectedFiles := make(map[*fileInfo]struct{}) - for _, entry := range blobs { - if processedBlobs.Has(entry.blob.BlobHandle) { - continue - } - for file := range entry.files { - affectedFiles[file] = struct{}{} - } - } - - for file := range affectedFiles { - if errFile := r.sanitizeError(file, err); errFile != nil { - return errFile - } - } - } - - return nil + return r.reportError(blobs, processedBlobs, err) } func (r *fileRestorer) sanitizeError(file *fileInfo, err error) error { @@ -273,6 +254,30 @@ func (r *fileRestorer) sanitizeError(file *fileInfo, err error) error { return err } +func (r *fileRestorer) reportError(blobs blobToFileOffsetsMapping, processedBlobs restic.BlobSet, err error) error { + if err == nil { + return nil + } + + // only report error for not yet processed blobs + affectedFiles := make(map[*fileInfo]struct{}) + for _, entry := range blobs { + if processedBlobs.Has(entry.blob.BlobHandle) { + continue + } + for file := range entry.files { + affectedFiles[file] = struct{}{} + } + } + + for file := range affectedFiles { + if errFile := r.sanitizeError(file, err); errFile != nil { + return errFile + } + } + return nil +} + func (r *fileRestorer) downloadBlobs(ctx context.Context, packID restic.ID, blobs blobToFileOffsetsMapping, processedBlobs restic.BlobSet) error { From e78be75d1efdf51051d1721e5bb48d041bc03ddb Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 7 Jan 2024 12:17:35 +0100 Subject: [PATCH 29/31] restore: separately restore blobs that are frequently referenced Writing these blobs to their files can take a long time and consequently cause the backend connection to time out. Avoid that by retrieving these blobs separately. --- internal/restorer/filerestorer.go | 27 +++++++++++++++++++++++++- internal/restorer/filerestorer_test.go | 21 ++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/internal/restorer/filerestorer.go b/internal/restorer/filerestorer.go index f2e2cf24a..99a460321 100644 --- a/internal/restorer/filerestorer.go +++ b/internal/restorer/filerestorer.go @@ -242,8 +242,33 @@ func (r *fileRestorer) downloadPack(ctx context.Context, pack *packInfo) error { // track already processed blobs for precise error reporting processedBlobs := restic.NewBlobSet() - err := r.downloadBlobs(ctx, pack.id, blobs, processedBlobs) + for _, entry := range blobs { + occurrences := 0 + for _, offsets := range entry.files { + occurrences += len(offsets) + } + // With a maximum blob size of 8MB, the normal blob streaming has to write + // at most 800MB for a single blob. This should be short enough to avoid + // network connection timeouts. Based on a quick test, a limit of 100 only + // selects a very small number of blobs (the number of references per blob + // - aka. `count` - seem to follow a expontential distribution) + if occurrences > 100 { + // process frequently referenced blobs first as these can take a long time to write + // which can cause backend connections to time out + delete(blobs, entry.blob.ID) + partialBlobs := blobToFileOffsetsMapping{entry.blob.ID: entry} + err := r.downloadBlobs(ctx, pack.id, partialBlobs, processedBlobs) + if err := r.reportError(blobs, processedBlobs, err); err != nil { + return err + } + } + } + if len(blobs) == 0 { + return nil + } + + err := r.downloadBlobs(ctx, pack.id, blobs, processedBlobs) return r.reportError(blobs, processedBlobs, err) } diff --git a/internal/restorer/filerestorer_test.go b/internal/restorer/filerestorer_test.go index 94b068159..c5bc3fe31 100644 --- a/internal/restorer/filerestorer_test.go +++ b/internal/restorer/filerestorer_test.go @@ -248,6 +248,27 @@ func TestFileRestorerPackSkip(t *testing.T) { } } +func TestFileRestorerFrequentBlob(t *testing.T) { + tempdir := rtest.TempDir(t) + + for _, sparse := range []bool{false, true} { + blobs := []TestBlob{ + {"data1-1", "pack1-1"}, + } + for i := 0; i < 10000; i++ { + blobs = append(blobs, TestBlob{"a", "pack1-1"}) + } + blobs = append(blobs, TestBlob{"end", "pack1-1"}) + + restoreAndVerify(t, tempdir, []TestFile{ + { + name: "file1", + blobs: blobs, + }, + }, nil, sparse) + } +} + func TestErrorRestoreFiles(t *testing.T) { tempdir := rtest.TempDir(t) content := []TestFile{ From 4ea3796455a3bb43dba03c268fbe3d9e787d133c Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sun, 7 Jan 2024 14:18:20 +0100 Subject: [PATCH 30/31] add changelog for reliable restores --- changelog/unreleased/pull-4626 | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 changelog/unreleased/pull-4626 diff --git a/changelog/unreleased/pull-4626 b/changelog/unreleased/pull-4626 new file mode 100644 index 000000000..ea16d749f --- /dev/null +++ b/changelog/unreleased/pull-4626 @@ -0,0 +1,11 @@ +Bugfix: Improve reliability of restoring large files + +In some cases restic failed to restore large files that frequently contain the +same file chunk. In combination with certain backends, this could result in +network connection timeouts that caused incomplete restores. + +Restic now includes special handling for such file chunks to ensure reliable +restores. + +https://github.com/restic/restic/pull/4626 +https://forum.restic.net/t/errors-restoring-with-restic-on-windows-server-s3/6943 From 2e8de9edfd15b88f59569fa5d22c5a950fc0bf7a Mon Sep 17 00:00:00 2001 From: Michael Eischer Date: Sat, 6 Jan 2024 17:31:16 +0100 Subject: [PATCH 31/31] rclone: Workaround for incorrect "not found" errors while listing files rclone returns a "not found" error if an internal error occurs while listing a folder. Ignoring this error lets restic erroneously think that there are no files, which can cause `prune` to wipe the whole repository. --- changelog/unreleased/issue-4612 | 11 +++++++++++ internal/backend/rest/rest.go | 9 +++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) create mode 100644 changelog/unreleased/issue-4612 diff --git a/changelog/unreleased/issue-4612 b/changelog/unreleased/issue-4612 new file mode 100644 index 000000000..ed99f4767 --- /dev/null +++ b/changelog/unreleased/issue-4612 @@ -0,0 +1,11 @@ +Bugfix: Improve error handling for `rclone` backend + +Since restic 0.16.0, if rclone encountered an error while listing files, +this could in rare circumstances cause restic to assume that there are no +files. Although unlikely, this situation could result in data loss if it +were to happen right when the `prune` command is listing existing snapshots. + +Error handling has now been improved to detect and work around this case. + +https://github.com/restic/restic/issues/4612 +https://github.com/restic/restic/pull/4618 diff --git a/internal/backend/rest/rest.go b/internal/backend/rest/rest.go index 1d1769b56..5310eba7c 100644 --- a/internal/backend/rest/rest.go +++ b/internal/backend/rest/rest.go @@ -328,8 +328,13 @@ func (b *Backend) List(ctx context.Context, t backend.FileType, fn func(backend. } if resp.StatusCode == http.StatusNotFound { - // ignore missing directories - return nil + if !strings.HasPrefix(resp.Header.Get("Server"), "rclone/") { + // ignore missing directories, unless the server is rclone. rclone + // already ignores missing directories, but misuses "not found" to + // report certain internal errors, see + // https://github.com/rclone/rclone/pull/7550 for details. + return nil + } } if resp.StatusCode != http.StatusOK {