mirror of https://github.com/restic/restic.git
Compare commits
671 Commits
Author | SHA1 | Date |
---|---|---|
Michael Eischer | 660679c2f6 | |
Michael Eischer | 456ebfb0c7 | |
Michael Eischer | 830e87059a | |
Michael Eischer | a26beb9c1c | |
dependabot[bot] | 46b0fac11e | |
dependabot[bot] | fced29b479 | |
dependabot[bot] | 05d7885a87 | |
Michael Eischer | 38654a3bd7 | |
Michael Eischer | 2280fbfd2e | |
Michael Eischer | 7e0ee5974f | |
Srigovind Nayak | de7b418bbe | |
Michael Eischer | c103c0830e | |
Michael Eischer | 513135b238 | |
Michael Eischer | cdd210185d | |
Michael Eischer | 5649334099 | |
Michael Eischer | 6869bdaaa8 | |
Michael Eischer | 13127f3f6b | |
Michael Eischer | 939b537c80 | |
Michael Eischer | 7b4f81d964 | |
Michael Eischer | a5c7a83470 | |
Michael Eischer | 52b3fee42b | |
Michael Eischer | e95cfff923 | |
Michael Eischer | fe8fbcc5ed | |
Michael Eischer | 3d4a620089 | |
greatroar | 46f04bc5f0 | |
Michael Eischer | a651af70d6 | |
Michael Eischer | 1c6067d93d | |
Michael Eischer | 21ce03cff2 | |
Michael Eischer | 0ad0b7ca7c | |
Michael Eischer | 619ce8bfa2 | |
Michael Eischer | 107f18849c | |
Michael Eischer | 18990dbed8 | |
Michael Eischer | 3828313974 | |
Michael Eischer | 04c181dbd0 | |
Michael Eischer | 21528c9f02 | |
Shivashis Padhi | 0271bb97f8 | |
Michael Eischer | 2c7ebf4c7f | |
Michael Eischer | 7ea508e7b8 | |
Michael Eischer | 496e57f956 | |
Michael Eischer | 5e0ea8fcfa | |
Michael Eischer | 50ec408302 | |
Michael Eischer | 58bbb27b4c | |
Michael Eischer | c5260373ca | |
Michael Eischer | b2f7a878cd | |
Michael Eischer | f20bf5eeba | |
Michael Eischer | 8e5d7d719c | |
Michael Eischer | 80132e71d8 | |
Michael Eischer | 130506250f | |
Michael Eischer | 1d2277b4c3 | |
Michael Eischer | d4b0d21199 | |
Michael Eischer | 55cb8d174a | |
Michael Eischer | 436afbff23 | |
Michael Eischer | 3c7b7efdc9 | |
Michael Eischer | 462b82a060 | |
Michael Eischer | 77873f5a9d | |
Michael Eischer | 2033c02b09 | |
Michael Eischer | 93098e9265 | |
Michael Eischer | ff4775a15f | |
Michael Eischer | 860b595a8b | |
Michael Eischer | f680a2331d | |
Michael Eischer | 027cc64737 | |
Michael Eischer | e52033a8bd | |
Michael Eischer | 57d69aa640 | |
Michael Eischer | 2ca1c37721 | |
Michael Eischer | 5f7b48e65f | |
Michael Eischer | ad98fbf7dd | |
Michael Eischer | 72482ce5bd | |
Michael Eischer | 68fa0e0305 | |
Michael Eischer | 9aa0c90fb2 | |
Michael Eischer | 76e6719f2e | |
Michael Eischer | 04ad9f0c0c | |
Michael Eischer | 550d1eeac3 | |
Michael Eischer | fb59e00614 | |
Michael Eischer | 447b486c20 | |
Michael Eischer | 6ca12c1b4a | |
Michael Eischer | 864995271e | |
Michael Eischer | 1266a4932f | |
Michael Eischer | e848ad651a | |
Michael Eischer | 0bb0720348 | |
Michael Eischer | c01bcb1001 | |
Michael Eischer | 0aa5c53842 | |
Michael Eischer | 8f1e70cd9b | |
Michael Eischer | 4df887406f | |
Michael Eischer | 3eeb6723cd | |
Michael Eischer | 3cc6827f09 | |
Michael Eischer | 7948912b31 | |
Michael Eischer | 3f46808898 | |
Michael Eischer | 82ca0030b7 | |
Michael Eischer | 57f9739573 | |
Michael Eischer | 5b7952e426 | |
Michael Eischer | c0e1f36830 | |
Michael Eischer | d106ad6921 | |
Michael Eischer | 16ef4d515b | |
Michael Eischer | e4a48085ae | |
Michael Eischer | 723247c8e5 | |
Michael Eischer | b1266867d2 | |
Michael Eischer | 98709a4372 | |
Michael Eischer | 512cd6ef07 | |
Michael Eischer | a60ee9b764 | |
Michael Eischer | a3633cad9e | |
Michael Eischer | b9cbf623fa | |
greatroar | 66d03c797e | |
greatroar | 0b56214473 | |
greatroar | 7f439a9c34 | |
Michael Eischer | 4021e67d97 | |
Michael Eischer | 8898f61717 | |
Michael Eischer | 5f23baabcc | |
Michael Eischer | 9c5bac6f25 | |
Michael Eischer | c56ecec9aa | |
Maik Riechert | 355f520936 | |
Michael Eischer | 1dfe1b8732 | |
Michael Eischer | 223aa22cb0 | |
Michael Eischer | 291c9677de | |
Michael Eischer | 673496b091 | |
Michael Eischer | 3d2410ed50 | |
Michael Eischer | d2c26e33f3 | |
Michael Eischer | 8a425c2f0a | |
Michael Eischer | aa4647f773 | |
Michael Eischer | 94e863885c | |
Michael Eischer | e40943a75d | |
Michael Eischer | 67e2ba0d40 | |
Michael Eischer | d8b184b3d3 | |
Michael Eischer | a1ca5e15c4 | |
Michael Eischer | 34d90aecf9 | |
Michael Eischer | ab9077bc13 | |
Michael Eischer | 8274f5b101 | |
Michael Eischer | 9795198189 | |
Michael Eischer | 0c1ba6d95d | |
Michael Eischer | eb6c653f89 | |
Michael Eischer | 74d90653e0 | |
Michael Eischer | 8f8d872a68 | |
Michael Eischer | ff0744b3af | |
Michael Eischer | 987c3b250c | |
Michael Eischer | bf16096771 | |
Michael Eischer | 4f45668b7c | |
Michael Eischer | ac805d6838 | |
Michael Eischer | 5214af88e2 | |
Michael Eischer | 3ff063e913 | |
Michael Eischer | 385cee09dc | |
Michael Eischer | e734746f75 | |
Michael Eischer | 97a307df1a | |
Michael Eischer | 8cce06d915 | |
Michael Eischer | 433a6aad29 | |
Michael Eischer | e401af07b2 | |
Michael Eischer | 7017adb7e9 | |
Michael Eischer | e33ce7f408 | |
Michael Eischer | 2ace242f36 | |
Michael Eischer | e9390352a7 | |
Michael Eischer | 503c8140b1 | |
Michael Eischer | 6563f1d2ca | |
Michael Eischer | 021fb49559 | |
Michael Eischer | 779c8d3527 | |
Michael Eischer | 1d6d3656b0 | |
Michael Eischer | 47232bf8b0 | |
Michael Eischer | dcd151147c | |
Michael Eischer | 53d15bcd1b | |
Michael Eischer | 394c8ca3ed | |
Michael Eischer | 6328b7e1f5 | |
Michael Eischer | 53561474d9 | |
Michael Eischer | aeb7eb245c | |
Michael Eischer | bf8cc59889 | |
Michael Eischer | 4740528a0b | |
Michael Eischer | 6a85df7297 | |
Michael Eischer | cfc420664a | |
Michael Eischer | d40f23e716 | |
Michael Eischer | e793c002ec | |
Michael Eischer | b4895ebd76 | |
Michael Eischer | eaa3f81d6b | |
Michael Eischer | c6d74458ee | |
Michael Eischer | 7ed560a201 | |
Michael Eischer | 92221c2a6d | |
Michael Eischer | b5fdb1d637 | |
Michael Eischer | e4f9bce384 | |
Michael Eischer | 3740700ddc | |
Michael Eischer | ebd01a4675 | |
Michael Eischer | 8778670232 | |
Michael Eischer | 0987c731ec | |
aneesh-n | a4fd1b91e5 | |
Michael Eischer | e184538ddf | |
Michael Eischer | 4d55a62ada | |
Michael Eischer | 7cce667f92 | |
Michael Eischer | bd03af2feb | |
Michael Eischer | 45509eafc8 | |
Michael Eischer | 24c1822220 | |
flow-c | d4477a5a99 | |
Michael Eischer | ffe5439149 | |
Michael Eischer | 676f0dc60d | |
Michael Eischer | 1e57057953 | |
Michael Eischer | 1ba0af6993 | |
Michael Eischer | ffc41ae62a | |
Michael Eischer | 4832c2fbfa | |
dependabot[bot] | 30609ae6b2 | |
dependabot[bot] | 502e5867a5 | |
dependabot[bot] | 18a6d6b408 | |
dependabot[bot] | 3bb88e8307 | |
aneesh-n | 672f6cd776 | |
aneesh-n | 08c6945d61 | |
Aneesh N | 3f76b902e5 | |
Michael Eischer | ccac7c7fb3 | |
DRON-666 | ccd35565ee | |
DRON-666 | 125dba23c5 | |
DRON-666 | 7ee889bb0d | |
DRON-666 | 90b168eb6c | |
DRON-666 | 24330c19a8 | |
DRON-666 | 5703e5a652 | |
DRON-666 | 0a8f9c5d9c | |
DRON-666 | 739d3243d9 | |
DRON-666 | bb0f93ef3d | |
DRON-666 | 3bac1f0135 | |
DRON-666 | 88c509e3e9 | |
DRON-666 | 9d3d915e2c | |
DRON-666 | 9182e6bab5 | |
DRON-666 | c4f67c0064 | |
DRON-666 | 7470e5356e | |
DRON-666 | 78dbc5ec58 | |
Michael Eischer | a1d682ce0e | |
Michael Eischer | 935327d480 | |
Michael Eischer | 669a669603 | |
Michael Eischer | faffd15d13 | |
Michael Eischer | 347e9d0765 | |
Altan Orhon | 871ea1eaf3 | |
Michael Eischer | a7b5e09902 | |
Michael Eischer | 3f9d50865d | |
Michael Eischer | 5f263752d7 | |
Michael Eischer | 484dbb1cf4 | |
Michael Eischer | 940a3159b5 | |
Michael Eischer | 31624aeffd | |
Michael Eischer | 910927670f | |
Michael Eischer | 6f2a4dea21 | |
Michael Eischer | 699ef5e9de | |
Michael Eischer | eb710a28e8 | |
Michael Eischer | 86c7909f41 | |
Michael Eischer | 93135dc705 | |
Michael Eischer | 21a7cb405c | |
Michael Eischer | b15d867414 | |
Michael Eischer | 2e6c43c695 | |
Michael Eischer | f7632de3d6 | |
Michael Eischer | 20d8eed400 | |
Michael Eischer | cf700d8794 | |
Michael Eischer | 666a0b0bdb | |
Michael Eischer | 621012dac0 | |
Michael Eischer | 6c6dceade3 | |
Michael Eischer | 10355c3fb6 | |
Michael Eischer | 228b35f074 | |
will-ca | 6aced61c72 | |
Michael Eischer | 4d22412e0c | |
coderwander | a82ed71de7 | |
Michael Eischer | 2173c69280 | |
Michael Eischer | 001bb71676 | |
Michael Eischer | c9191ea72c | |
Michael Eischer | 09587e6c08 | |
Michael Eischer | defd7ae729 | |
Michael Eischer | 038586dc9d | |
Michael Eischer | d8622c86eb | |
Michael Eischer | 8d507c1372 | |
Michael Eischer | 310db03c0e | |
Michael Eischer | 7d1b9cde34 | |
Michael Eischer | b25fc2c89d | |
Michael Eischer | c65459cd8a | |
Michael Eischer | eda9f7beb4 | |
Michael Eischer | 35277b7797 | |
Michael Eischer | 7ba5e95a82 | |
Michael Eischer | 4c9a10ca37 | |
Michael Eischer | 85e4021619 | |
Michael Eischer | 55d56db31b | |
Michael Eischer | fc3b548625 | |
Michael Eischer | df9d4b455d | |
Michael Eischer | 866ddf5698 | |
Michael Eischer | 32a234b67e | |
Michael Eischer | 739d11c2eb | |
Michael Eischer | 591b421c4a | |
Michael Eischer | 8efc3a8b7d | |
Michael Eischer | bf054c09d2 | |
Michael Eischer | 0747cf5319 | |
Michael Eischer | 6091029fd6 | |
Martin Geisler | 09d2183351 | |
Michael Eischer | a4b7ebecfc | |
dependabot[bot] | ba136b31b8 | |
Michael Eischer | f328111a6e | |
Michael Eischer | 9fb017e67a | |
Michael Eischer | 49f98f25fc | |
dependabot[bot] | 96c602a6de | |
Michael Eischer | 1d0a20dd58 | |
Michael Eischer | 6cca1d5705 | |
dependabot[bot] | f8a72ac2a3 | |
Michael Eischer | 5145c8f9c0 | |
Michael Eischer | 831fc4413d | |
Stephan Paul | df07814ec2 | |
Michael Eischer | ec2b79834a | |
Michael Eischer | 510f6f06b0 | |
Michael Eischer | 07eb6c315b | |
Michael Eischer | 5e98f1e2eb | |
Michael Eischer | 8155dbe711 | |
Michael Eischer | d18726cd70 | |
Michael Eischer | dc441c57a7 | |
Michael Eischer | 3ba1fa3cee | |
Michael Eischer | 044e8bf821 | |
Michael Eischer | e8df50fa3c | |
Michael Eischer | cbb5f89252 | |
Michael Eischer | 118a69a84b | |
Michael Eischer | 7f9ad1c3db | |
Michael Eischer | 71b6284155 | |
Michael Eischer | cf81f8ced6 | |
Michael Eischer | 21cf38fe96 | |
Michael Eischer | d705741571 | |
Michael Eischer | a9b3d86c4f | |
Michael Eischer | a26d6ffa72 | |
Michael Eischer | 2ba21fe72b | |
Michael Eischer | 870904d3ae | |
Facundo Tuesca | 15555c9898 | |
Michael Eischer | 63a2350c9e | |
Michael Eischer | 8876e3025b | |
Michael Eischer | 4f4979f4e8 | |
Michael Eischer | 1497525e15 | |
Michael Eischer | a8face3a25 | |
Michael Eischer | aee6d311f1 | |
Michael Eischer | 6ac7519188 | |
Michael Eischer | add37fcd9f | |
Adam Eijdenberg | 6e775d3787 | |
Michael Eischer | 5c4a4b4a30 | |
Michael Eischer | d1d773cfcd | |
Facundo Tuesca | 521713fc94 | |
avoidalone | ac948fccda | |
rawtaz | 9284f7413a | |
rawtaz | 1287b977b4 | |
Leo Heitmann Ruiz | 00f762373f | |
Leo Heitmann Ruiz | 9f3e1462c0 | |
Michael Eischer | 69ca12d2eb | |
Michael Eischer | 98a6817d01 | |
Michael Eischer | f8852f0eb6 | |
Michael Eischer | 1a8bf358f1 | |
Michael Eischer | 396a61a992 | |
Michael Eischer | a9b64cd7ad | |
Michael Eischer | fe68d2cafb | |
Michael Eischer | 70839155f2 | |
Michael Eischer | 1c77c51a03 | |
Michael Eischer | 5974a79497 | |
Michael Eischer | 0589da60b3 | |
dependabot[bot] | 608116817b | |
Michael Eischer | f742da8b2f | |
Michael Eischer | af1684743f | |
Michael Eischer | 5b9de4d8b7 | |
Michael Eischer | fceef67abe | |
Michael Eischer | 8506cae710 | |
Michael Eischer | 87d47ef189 | |
Michael Eischer | 55abf25ea8 | |
Srigovind Nayak | b48b1fa2c9 | |
dependabot[bot] | 8e7f29ae28 | |
dependabot[bot] | 79e8ddac3f | |
dependabot[bot] | b5a9b5d0bc | |
dependabot[bot] | f185c80cf0 | |
dependabot[bot] | 70c8aaa303 | |
dependabot[bot] | e1a588b75c | |
Michael Eischer | e71660cd1e | |
Aneesh Nireshwalia | 062d408987 | |
Aneesh Nireshwalia | 5764300022 | |
Aneesh Nireshwalia | c0a1b9ada5 | |
Aneesh Nireshwalia | 90916f53de | |
Aneesh Nireshwalia | 70cf8e3788 | |
Aneesh Nireshwalia | e3e59fef24 | |
Aneesh Nireshwalia | 09ce1b4e58 | |
Michael Eischer | 6a13e451b1 | |
Michael Eischer | a8f5684f68 | |
Michael Eischer | 681395955e | |
Michael Eischer | b6520038fd | |
Michael Eischer | 38f91d3b5e | |
Michael Eischer | 86897314d5 | |
Michael Eischer | a59f654fa6 | |
Michael Eischer | 8b1a85711f | |
Michael Eischer | b953dc8f58 | |
Aneesh Nireshwalia | e8211cb64a | |
Aneesh Nireshwalia | 4bbd25a37f | |
Aneesh Nireshwalia | d4be734c73 | |
Aneesh Nireshwalia | eeb1aa5388 | |
Aneesh Nireshwalia | 0962917974 | |
Aneesh Nireshwalia | 62a8a599f1 | |
Aneesh Nireshwalia | 94de87d4b7 | |
Michael Eischer | c6311c1e32 | |
Michael Eischer | 0a65a0f94f | |
Brian Harring | b41107dcaf | |
Brian Harring | 30e979d252 | |
Michael Eischer | cfbeb2cde5 | |
Michael Eischer | 80754dbf0c | |
Michael Eischer | 4c3218ef9f | |
Michael Eischer | 18b0bbbf42 | |
Michael Eischer | 6fbb470835 | |
Michael Eischer | 0a36d193d8 | |
Michael Eischer | 69304cd74f | |
Michael Eischer | c3b0e6d004 | |
Michael Eischer | 9e3703ded5 | |
Michael Eischer | 527a3ff2b2 | |
Michael Eischer | ed4a4f8748 | |
Michael Eischer | 4073299a7c | |
Michael Eischer | 6397615fbb | |
Michael Eischer | 544fe38786 | |
Michael Eischer | 772e3416d1 | |
Michael Eischer | 22a3cea1b3 | |
Michael Eischer | 19bf2cf52d | |
Michael Eischer | 5b5d506472 | |
Michael Eischer | dde556e8e8 | |
Michael Eischer | ee1ff3c1d0 | |
Michael Eischer | 667a2f5369 | |
Michael Eischer | 2ab18a92e6 | |
Alexander Neumann | c0514dd8ba | |
Alexander Neumann | a8cda0119c | |
Alexander Neumann | 9720935c56 | |
Michael Eischer | 68cc327b15 | |
Michael Eischer | 15d6fa1f83 | |
lou | 80db02fc35 | |
Michael Eischer | 6a2b10e2a8 | |
Michael Eischer | e46b21ab80 | |
Michael Eischer | eb389a2d25 | |
Srigovind Nayak | 795d33b3ee | |
Michael Eischer | 0cffdb7493 | |
Michael Eischer | f5ffa40652 | |
Srigovind Nayak | 175c14b5c9 | |
Michael Eischer | bca099ac7f | |
Michael Eischer | 0f09a8870c | |
Srigovind Nayak | 5771c4ecfb | |
Michael Eischer | b63bfd2257 | |
Alexander Neumann | 0f9fa44de5 | |
Michael Eischer | d5e662315a | |
Michael Eischer | effe76aaf5 | |
Michael Eischer | 5957417b1f | |
Michael Eischer | 219d8e3c18 | |
Michael Eischer | a737fe1e47 | |
Michael Eischer | 86b38a0b17 | |
Michael Eischer | 7d31180fe6 | |
Michael Eischer | c32e5e2abb | |
Michael Eischer | c97a271e89 | |
Michael Eischer | 66e8971659 | |
Michael Eischer | 193140525c | |
Michael Eischer | 96518d7c4a | |
Michael Eischer | 2dbb18128c | |
Michael Eischer | 30a84e9003 | |
Michael Eischer | c01a0c6da7 | |
Michael Eischer | 16e3f79e8b | |
Michael Eischer | bb92b487f7 | |
dependabot[bot] | cf7cad11de | |
Michael Eischer | 370d9c31f4 | |
Michael Eischer | 6581133e85 | |
Michael Eischer | 207a4a5e8e | |
Michael Eischer | cbf9cd4a7f | |
dependabot[bot] | 552f01662b | |
dependabot[bot] | 7f5ea511bc | |
Joram Berger | b07afa9b02 | |
Michael Eischer | 8b08b522c9 | |
Michael Eischer | eaf9659efc | |
Michael Eischer | ba136ff60c | |
Lionel Sausin | 8fbe328371 | |
Michael Eischer | 4273e06a43 | |
Michael Eischer | 248c144f72 | |
Michael Eischer | 5dca8a70d5 | |
Michael Eischer | 765729d009 | |
Michael Eischer | a09d51d96c | |
Michael Eischer | e44e4b00a6 | |
Michael Eischer | 10e71af759 | |
Michael Eischer | c90f24a06c | |
Michael Eischer | d4ed7c8858 | |
Michael Eischer | 2c80cfa4a5 | |
Michael Eischer | 261737abc8 | |
Michael Eischer | a2f2f8fb4c | |
Michael Eischer | 4bae54d040 | |
Michael Eischer | 509b339d54 | |
Michael Eischer | a2fe337610 | |
Michael Eischer | 1b008c92d3 | |
Michael Eischer | 9ecbda059c | |
Nils Decker | b2703a4089 | |
Nils Decker | a9310948cf | |
Michael Eischer | 246559e654 | |
Michael Eischer | 1dfd854769 | |
Michael Eischer | bfb56b78e1 | |
Michael Eischer | 3424088274 | |
Michael Eischer | 724ec179e3 | |
Michael Eischer | f0e1ad2285 | |
Michael Eischer | fd579421dd | |
Michael Eischer | 42c9318b9c | |
Michael Eischer | 764b0bacd6 | |
Michael Eischer | 7c351bc53c | |
Michael Eischer | feeab84204 | |
Michael Eischer | d7a50fe739 | |
Michael Eischer | 6b65a495b1 | |
Michael Eischer | d26d2d41f8 | |
Michael Eischer | cb50832d50 | |
Michael Eischer | bedff1ed6d | |
Michael Eischer | c13bf0b607 | |
Michael Eischer | 25ac1549e7 | |
Michael Eischer | ae9683336d | |
Michael Eischer | 446167ae80 | |
Michael Eischer | 5b36c4eb5f | |
Michael Eischer | 1419baf67a | |
Michael Eischer | 66103aea3d | |
Michael Eischer | 79f2939eb9 | |
Michael Eischer | 0e2ee06803 | |
Michael Eischer | 2927982256 | |
Michael Eischer | 6cc2bec5dd | |
Michael Eischer | 18806944f6 | |
adrian5 | 609f84e095 | |
Michael Eischer | 767c2539a0 | |
Michael Eischer | 6bdca13603 | |
Michael Eischer | f1f34eb3e5 | |
Michael Eischer | fee83e1c09 | |
Michael Eischer | 6696195f38 | |
Michael Eischer | a763a5c67d | |
Vladislav Belous | 8ca58b487c | |
Michael Eischer | 62111f4379 | |
Michael Eischer | 2c310a526e | |
Michael Eischer | 6b7b5c89e9 | |
Michael Eischer | 22d0c3f8dc | |
Michael Eischer | fb422497af | |
Michael Eischer | 54c5c72e5a | |
Michael Eischer | 5f49eec655 | |
Michael Eischer | ec13105093 | |
Michael Eischer | bd883caae1 | |
Michael Eischer | b1a8fd1d03 | |
Michael Eischer | fdcbb53017 | |
Michael Eischer | 0b39940fdb | |
Michael Eischer | 147b0e54cb | |
Ben Northway | 5413877d33 | |
Michael Eischer | 03e06d0797 | |
Michael Eischer | 7b2de84763 | |
Michael Eischer | c31e9418ba | |
Michael Eischer | 2e8de9edfd | |
Michael Eischer | 4ea3796455 | |
Michael Eischer | e78be75d1e | |
Michael Eischer | 2267910418 | |
Michael Eischer | 00d18b7a88 | |
Michael Eischer | 9328f34d43 | |
Michael Eischer | 77434c6e2b | |
Michael Eischer | 4248c6c3ca | |
Michael Eischer | e4a7eb09ef | |
Michael Eischer | f8b4e932ef | |
Michael Eischer | 100872308f | |
Michael Eischer | dac3508170 | |
Michael Eischer | 77b1c52673 | |
Michael Eischer | fe5c337ca2 | |
Michael Eischer | 3e29f8dddf | |
Michael Eischer | 76f507c775 | |
Michael Eischer | 6ef23b401b | |
Michael Eischer | 62f99a3b2f | |
Michael Eischer | 0360e540af | |
Michael Eischer | e6dfefba13 | |
Michael Eischer | 02bc73f5eb | |
Michael Eischer | 20cf4777cb | |
Erik Kristensen | 5ffb536aae | |
Michael Eischer | 1604922360 | |
Michael Eischer | c7844530d8 | |
Michael Eischer | 33b7c84a7a | |
Michael Manganiello | 045aa64558 | |
Michael Eischer | b2b7669ca0 | |
Michael Eischer | 4f6b1bb6f6 | |
Michael Eischer | 3549635243 | |
Daniel Danner | a7dc18e697 | |
Michael Eischer | 51419c51d3 | |
Michael Eischer | 6b79834cc8 | |
Michael Eischer | 0018bb7854 | |
Michael Eischer | 634e2a46d9 | |
Michael Eischer | dfcab92db2 | |
dependabot[bot] | 3666eef76c | |
Michael Eischer | 3a61622dfe | |
Michael Eischer | 5c4fca76df | |
Michael Eischer | 98da0bdd12 | |
dependabot[bot] | 2c60dd97ae | |
dependabot[bot] | 40905403f4 | |
dependabot[bot] | 7e7cbe8e19 | |
dependabot[bot] | 44646c20be | |
Michael Eischer | 8f9a35779e | |
Michael Eischer | 23e1b4bbb1 | |
Michael Eischer | 01b33734ab | |
Michael Eischer | 649a6409ee | |
Michael Eischer | c31f5f986c | |
Michael Eischer | 2730d05fce | |
Michael Eischer | 893d0d6325 | |
Gabriel Kabbe | 7de97d7480 | |
Gabriel Kabbe | 004520a238 | |
Gabriel Kabbe | a02d8d75c2 | |
Gabriel Kabbe | 7bf38b6c50 | |
Gabriel Kabbe | da1704b2d5 | |
Gabriel Kabbe | 3026baea07 | |
Michael Eischer | 1196c72819 | |
Michael Eischer | 433dd92959 | |
Markus Zoppelt | c14740c50f | |
Michael Eischer | 5537460664 | |
Sébastien Gross | f7587be28f | |
Michael Eischer | 91fb703756 | |
Michael Eischer | d7ff862b8d | |
mmattel | db1d920c80 | |
Michael Eischer | c6299f8dbd | |
Giuseppe D'Andrea | a128976014 | |
Michael Eischer | e2f6109a52 | |
Michael Eischer | 30e6ed038c | |
Michael Eischer | e96d1ee33e | |
dependabot[bot] | 0054db394f | |
Michael Eischer | 53ebe91a50 | |
Michael Eischer | 9ceaea34dd | |
Michael Eischer | 356b7aac16 | |
Joda Stößer | eef7c65655 | |
Michael Eischer | 97b8629336 | |
Michael Eischer | d2ecd6bef2 | |
Michael Eischer | 634750a732 | |
Quang-Linh LE | c554825e2d | |
Michael Kuhn | fd2fb233aa | |
Michael Eischer | da4e3edbbc | |
Michael Eischer | dbbd31bc3a | |
Joram Berger | 12af20e606 | |
Andrea Gelmini | 241916d55b | |
dependabot[bot] | 427b90cf82 | |
dependabot[bot] | eec6e014f4 | |
dependabot[bot] | fa46a47e22 | |
Michael Eischer | b72de5a883 | |
Michael Eischer | 6086ae4ca7 | |
Scott Leggett | aeaf527be1 | |
Michael Eischer | 19068aa82f | |
Michael Eischer | 81ca9d28f2 | |
Tobias Speicher | ce53ea32c6 | |
Tobias Speicher | 10cbc169c1 | |
DRON-666 | 03f8f494e9 | |
Tobias Speicher | ab23d033b6 | |
Tobias Speicher | 6f1efcb28b | |
Michael Eischer | 9c399e55e3 | |
Michael Eischer | e550bc0713 | |
Michael Eischer | 28aa9826af | |
dependabot[bot] | 6dde019ac8 | |
dependabot[bot] | c2f9e21d3c | |
dependabot[bot] | 67e6b9104a | |
Tobias Speicher | 6ca07ee004 | |
Tobias Speicher | d45cc52468 | |
Michael Eischer | 42ab3ea2b9 | |
Michael Eischer | be28a02626 | |
Michael Eischer | 5d152c7720 | |
Michael Eischer | ee305e6041 | |
Michael Eischer | 8bceb8e359 | |
Michael Eischer | 317144c1d6 | |
Michael Eischer | 7d879705ad | |
Enrico204 | 37a312e505 | |
Enrico204 | c0ca54dc8a | |
Enrico204 | 81f8d473df | |
Enrico204 | 6990b0122e | |
Enrico204 | 072b227544 | |
Enrico204 | 4e5caab114 | |
Sebastian Hoß | c133065a9f | |
Sebastian Hoß | 25350a9c55 | |
Sebastian Hoß | a2b76ff34f | |
Sebastian Hoß | 333fe1c3cf | |
Sebastian Hoß | a8657bde68 | |
Michael Eischer | 104107886a | |
Michael Eischer | 731b3a4357 | |
Michael Eischer | a8fdcf79b7 | |
Michael Eischer | 45962c2847 | |
Michael Eischer | 50ef01131a | |
rawtaz | 6be3a8fe51 | |
Michael Eischer | 5166bde386 | |
Leo R. Lundgren | aafb806a8c | |
Martin Michlmayr | 41e6a02bcc | |
Martin Michlmayr | b51fe2fb69 | |
Michael Eischer | 56537fb48e | |
Michael Eischer | feea567868 | |
Michael Eischer | 2968b52f84 | |
Michael Eischer | 619e80d7cc | |
Michael Eischer | 3804e50d64 | |
Michael Eischer | c19e39968f | |
Michael Eischer | 550be5c1e9 | |
Michael Eischer | 249605843b | |
Michael Eischer | c7b770eb1f | |
Michael Eischer | 1b8a67fe76 | |
Michael Eischer | ceb0774af1 | |
Michael Eischer | b6d79bdf6f | |
Michael Eischer | 7881309d63 | |
Michael Eischer | 8e6fdf5edf | |
Michael Eischer | c2e3e8d6ea | |
Arash Farr | d15ffd9c92 |
|
@ -25,7 +25,7 @@ jobs:
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Log in to the Container registry
|
- name: Log in to the Container registry
|
||||||
uses: docker/login-action@3d58c274f17dffee475a5520cbe67f0a882c4dbb
|
uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446
|
||||||
with:
|
with:
|
||||||
registry: ${{ env.REGISTRY }}
|
registry: ${{ env.REGISTRY }}
|
||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
|
@ -33,7 +33,7 @@ jobs:
|
||||||
|
|
||||||
- name: Extract metadata (tags, labels) for Docker
|
- name: Extract metadata (tags, labels) for Docker
|
||||||
id: meta
|
id: meta
|
||||||
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
|
uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81
|
||||||
with:
|
with:
|
||||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||||
tags: |
|
tags: |
|
||||||
|
@ -45,7 +45,7 @@ jobs:
|
||||||
uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3
|
uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
- name: Set up Docker Buildx
|
||||||
uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226
|
uses: docker/setup-buildx-action@2b51285047da1547ffb1b2203d8be4c0af6b1f20
|
||||||
|
|
||||||
- name: Ensure consistent binaries
|
- name: Ensure consistent binaries
|
||||||
run: |
|
run: |
|
||||||
|
@ -55,7 +55,7 @@ jobs:
|
||||||
if: github.ref != 'refs/heads/master'
|
if: github.ref != 'refs/heads/master'
|
||||||
|
|
||||||
- name: Build and push Docker image
|
- name: Build and push Docker image
|
||||||
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
|
uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0
|
||||||
with:
|
with:
|
||||||
push: true
|
push: true
|
||||||
context: .
|
context: .
|
||||||
|
|
|
@ -13,7 +13,7 @@ permissions:
|
||||||
contents: read
|
contents: read
|
||||||
|
|
||||||
env:
|
env:
|
||||||
latest_go: "1.21.x"
|
latest_go: "1.22.x"
|
||||||
GO111MODULE: on
|
GO111MODULE: on
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
|
@ -23,27 +23,32 @@ jobs:
|
||||||
# list of jobs to run:
|
# list of jobs to run:
|
||||||
include:
|
include:
|
||||||
- job_name: Windows
|
- job_name: Windows
|
||||||
go: 1.21.x
|
go: 1.22.x
|
||||||
os: windows-latest
|
os: windows-latest
|
||||||
|
|
||||||
- job_name: macOS
|
- job_name: macOS
|
||||||
go: 1.21.x
|
go: 1.22.x
|
||||||
os: macOS-latest
|
os: macOS-latest
|
||||||
test_fuse: false
|
test_fuse: false
|
||||||
|
|
||||||
- job_name: Linux
|
- job_name: Linux
|
||||||
go: 1.21.x
|
go: 1.22.x
|
||||||
os: ubuntu-latest
|
os: ubuntu-latest
|
||||||
test_cloud_backends: true
|
test_cloud_backends: true
|
||||||
test_fuse: true
|
test_fuse: true
|
||||||
check_changelog: true
|
check_changelog: true
|
||||||
|
|
||||||
- job_name: Linux (race)
|
- job_name: Linux (race)
|
||||||
go: 1.21.x
|
go: 1.22.x
|
||||||
os: ubuntu-latest
|
os: ubuntu-latest
|
||||||
test_fuse: true
|
test_fuse: true
|
||||||
test_opts: "-race"
|
test_opts: "-race"
|
||||||
|
|
||||||
|
- job_name: Linux
|
||||||
|
go: 1.21.x
|
||||||
|
os: ubuntu-latest
|
||||||
|
test_fuse: true
|
||||||
|
|
||||||
- job_name: Linux
|
- job_name: Linux
|
||||||
go: 1.20.x
|
go: 1.20.x
|
||||||
os: ubuntu-latest
|
os: ubuntu-latest
|
||||||
|
@ -69,7 +74,7 @@ jobs:
|
||||||
- name: Get programs (Linux/macOS)
|
- name: Get programs (Linux/macOS)
|
||||||
run: |
|
run: |
|
||||||
echo "build Go tools"
|
echo "build Go tools"
|
||||||
go install github.com/restic/rest-server/cmd/rest-server@latest
|
go install github.com/restic/rest-server/cmd/rest-server@master
|
||||||
|
|
||||||
echo "install minio server"
|
echo "install minio server"
|
||||||
mkdir $HOME/bin
|
mkdir $HOME/bin
|
||||||
|
@ -101,7 +106,7 @@ jobs:
|
||||||
$ProgressPreference = 'SilentlyContinue'
|
$ProgressPreference = 'SilentlyContinue'
|
||||||
|
|
||||||
echo "build Go tools"
|
echo "build Go tools"
|
||||||
go install github.com/restic/rest-server/...
|
go install github.com/restic/rest-server/cmd/rest-server@master
|
||||||
|
|
||||||
echo "install minio server"
|
echo "install minio server"
|
||||||
mkdir $Env:USERPROFILE/bin
|
mkdir $Env:USERPROFILE/bin
|
||||||
|
@ -242,6 +247,10 @@ jobs:
|
||||||
lint:
|
lint:
|
||||||
name: lint
|
name: lint
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
# allow annotating code in the PR
|
||||||
|
checks: write
|
||||||
steps:
|
steps:
|
||||||
- name: Set up Go ${{ env.latest_go }}
|
- name: Set up Go ${{ env.latest_go }}
|
||||||
uses: actions/setup-go@v5
|
uses: actions/setup-go@v5
|
||||||
|
@ -252,10 +261,10 @@ jobs:
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: golangci-lint
|
- name: golangci-lint
|
||||||
uses: golangci/golangci-lint-action@v3
|
uses: golangci/golangci-lint-action@v6
|
||||||
with:
|
with:
|
||||||
# Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
|
# Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
|
||||||
version: v1.55.2
|
version: v1.57.1
|
||||||
args: --verbose --timeout 5m
|
args: --verbose --timeout 5m
|
||||||
|
|
||||||
# only run golangci-lint for pull requests, otherwise ALL hints get
|
# only run golangci-lint for pull requests, otherwise ALL hints get
|
||||||
|
@ -293,7 +302,7 @@ jobs:
|
||||||
|
|
||||||
- name: Docker meta
|
- name: Docker meta
|
||||||
id: meta
|
id: meta
|
||||||
uses: docker/metadata-action@v4
|
uses: docker/metadata-action@v5
|
||||||
with:
|
with:
|
||||||
# list of Docker images to use as base name for tags
|
# list of Docker images to use as base name for tags
|
||||||
images: |
|
images: |
|
||||||
|
@ -316,7 +325,7 @@ jobs:
|
||||||
|
|
||||||
- name: Build and push
|
- name: Build and push
|
||||||
id: docker_build
|
id: docker_build
|
||||||
uses: docker/build-push-action@v4
|
uses: docker/build-push-action@v5
|
||||||
with:
|
with:
|
||||||
push: false
|
push: false
|
||||||
context: .
|
context: .
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
/.idea
|
||||||
/restic
|
/restic
|
||||||
/restic.exe
|
/restic.exe
|
||||||
/.vagrant
|
/.vagrant
|
||||||
|
|
|
@ -35,6 +35,11 @@ linters:
|
||||||
# parse and typecheck code
|
# parse and typecheck code
|
||||||
- typecheck
|
- typecheck
|
||||||
|
|
||||||
|
# ensure that http response bodies are closed
|
||||||
|
- bodyclose
|
||||||
|
|
||||||
|
- importas
|
||||||
|
|
||||||
issues:
|
issues:
|
||||||
# don't use the default exclude rules, this hides (among others) ignored
|
# don't use the default exclude rules, this hides (among others) ignored
|
||||||
# errors from Close() calls
|
# errors from Close() calls
|
||||||
|
@ -51,3 +56,14 @@ issues:
|
||||||
# staticcheck: there's no easy way to replace these packages
|
# staticcheck: there's no easy way to replace these packages
|
||||||
- "SA1019: \"golang.org/x/crypto/poly1305\" is deprecated"
|
- "SA1019: \"golang.org/x/crypto/poly1305\" is deprecated"
|
||||||
- "SA1019: \"golang.org/x/crypto/openpgp\" is deprecated"
|
- "SA1019: \"golang.org/x/crypto/openpgp\" is deprecated"
|
||||||
|
|
||||||
|
exclude-rules:
|
||||||
|
# revive: ignore unused parameters in tests
|
||||||
|
- path: (_test\.go|testing\.go|backend/.*/tests\.go)
|
||||||
|
text: "unused-parameter:"
|
||||||
|
|
||||||
|
linters-settings:
|
||||||
|
importas:
|
||||||
|
alias:
|
||||||
|
- pkg: github.com/restic/restic/internal/test
|
||||||
|
alias: rtest
|
||||||
|
|
|
@ -8,6 +8,10 @@ build:
|
||||||
tools:
|
tools:
|
||||||
python: "3.11"
|
python: "3.11"
|
||||||
|
|
||||||
|
# Build HTMLZip
|
||||||
|
formats:
|
||||||
|
- htmlzip
|
||||||
|
|
||||||
# Build documentation in the docs/ directory with Sphinx
|
# Build documentation in the docs/ directory with Sphinx
|
||||||
sphinx:
|
sphinx:
|
||||||
configuration: doc/conf.py
|
configuration: doc/conf.py
|
||||||
|
|
|
@ -3488,7 +3488,7 @@ restic users. The changes are ordered by importance.
|
||||||
|
|
||||||
NOTE: This new implementation does not guarantee order in which blobs are
|
NOTE: This new implementation does not guarantee order in which blobs are
|
||||||
written to the target files and, for example, the last blob of a file can be
|
written to the target files and, for example, the last blob of a file can be
|
||||||
written to the file before any of the preceeding file blobs. It is therefore
|
written to the file before any of the preceding file blobs. It is therefore
|
||||||
possible to have gaps in the data written to the target files if restore fails
|
possible to have gaps in the data written to the target files if restore fails
|
||||||
or interrupted by the user.
|
or interrupted by the user.
|
||||||
|
|
||||||
|
|
|
@ -6,7 +6,8 @@ Ways to Help Out
|
||||||
Thank you for your contribution! Please **open an issue first** (or add a
|
Thank you for your contribution! Please **open an issue first** (or add a
|
||||||
comment to an existing issue) if you plan to work on any code or add a new
|
comment to an existing issue) if you plan to work on any code or add a new
|
||||||
feature. This way, duplicate work is prevented and we can discuss your ideas
|
feature. This way, duplicate work is prevented and we can discuss your ideas
|
||||||
and design first.
|
and design first. Small bugfixes are an exception to this rule, just open a
|
||||||
|
pull request in this case.
|
||||||
|
|
||||||
There are several ways you can help us out. First of all code contributions and
|
There are several ways you can help us out. First of all code contributions and
|
||||||
bug fixes are most welcome. However even "minor" details as fixing spelling
|
bug fixes are most welcome. However even "minor" details as fixing spelling
|
||||||
|
@ -61,7 +62,7 @@ uploading it somewhere or post only the parts that are really relevant.
|
||||||
If restic gets stuck, please also include a stacktrace in the description.
|
If restic gets stuck, please also include a stacktrace in the description.
|
||||||
On non-Windows systems, you can send a SIGQUIT signal to restic or press
|
On non-Windows systems, you can send a SIGQUIT signal to restic or press
|
||||||
`Ctrl-\` to achieve the same result. This causes restic to print a stacktrace
|
`Ctrl-\` to achieve the same result. This causes restic to print a stacktrace
|
||||||
and then exit immediatelly. This will not damage your repository, however,
|
and then exit immediately. This will not damage your repository, however,
|
||||||
it might be necessary to manually clean up stale lock files using
|
it might be necessary to manually clean up stale lock files using
|
||||||
`restic unlock`.
|
`restic unlock`.
|
||||||
|
|
||||||
|
|
18
README.md
18
README.md
|
@ -10,8 +10,7 @@ For detailed usage and installation instructions check out the [documentation](h
|
||||||
|
|
||||||
You can ask questions in our [Discourse forum](https://forum.restic.net).
|
You can ask questions in our [Discourse forum](https://forum.restic.net).
|
||||||
|
|
||||||
Quick start
|
## Quick start
|
||||||
-----------
|
|
||||||
|
|
||||||
Once you've [installed](https://restic.readthedocs.io/en/latest/020_installation.html) restic, start
|
Once you've [installed](https://restic.readthedocs.io/en/latest/020_installation.html) restic, start
|
||||||
off with creating a repository for your backups:
|
off with creating a repository for your backups:
|
||||||
|
@ -59,7 +58,7 @@ Therefore, restic supports the following backends for storing backups natively:
|
||||||
Restic is a program that does backups right and was designed with the
|
Restic is a program that does backups right and was designed with the
|
||||||
following principles in mind:
|
following principles in mind:
|
||||||
|
|
||||||
- **Easy:** Doing backups should be a frictionless process, otherwise
|
- **Easy**: Doing backups should be a frictionless process, otherwise
|
||||||
you might be tempted to skip it. Restic should be easy to configure
|
you might be tempted to skip it. Restic should be easy to configure
|
||||||
and use, so that, in the event of a data loss, you can just restore
|
and use, so that, in the event of a data loss, you can just restore
|
||||||
it. Likewise, restoring data should not be complicated.
|
it. Likewise, restoring data should not be complicated.
|
||||||
|
@ -92,20 +91,17 @@ reproduce a byte identical version from the source code for that
|
||||||
release. Instructions on how to do that are contained in the
|
release. Instructions on how to do that are contained in the
|
||||||
[builder repository](https://github.com/restic/builder).
|
[builder repository](https://github.com/restic/builder).
|
||||||
|
|
||||||
News
|
## News
|
||||||
----
|
|
||||||
|
|
||||||
You can follow the restic project on Mastodon [@resticbackup](https://fosstodon.org/@restic) or by subscribing to
|
You can follow the restic project on Mastodon [@resticbackup](https://fosstodon.org/@restic) or subscribe to
|
||||||
the [project blog](https://restic.net/blog/).
|
the [project blog](https://restic.net/blog/).
|
||||||
|
|
||||||
License
|
## License
|
||||||
-------
|
|
||||||
|
|
||||||
Restic is licensed under [BSD 2-Clause License](https://opensource.org/licenses/BSD-2-Clause). You can find the
|
Restic is licensed under [BSD 2-Clause License](https://opensource.org/licenses/BSD-2-Clause). You can find the
|
||||||
complete text in [``LICENSE``](LICENSE).
|
complete text in [`LICENSE`](LICENSE).
|
||||||
|
|
||||||
Sponsorship
|
## Sponsorship
|
||||||
-----------
|
|
||||||
|
|
||||||
Backend integration tests for Google Cloud Storage and Microsoft Azure Blob
|
Backend integration tests for Google Cloud Storage and Microsoft Azure Blob
|
||||||
Storage are sponsored by [AppsCode](https://appscode.com)!
|
Storage are sponsored by [AppsCode](https://appscode.com)!
|
||||||
|
|
|
@ -10,7 +10,7 @@ https://github.com/restic/restic/issues/2244
|
||||||
|
|
||||||
NOTE: This new implementation does not guarantee order in which blobs
|
NOTE: This new implementation does not guarantee order in which blobs
|
||||||
are written to the target files and, for example, the last blob of a
|
are written to the target files and, for example, the last blob of a
|
||||||
file can be written to the file before any of the preceeding file blobs.
|
file can be written to the file before any of the preceding file blobs.
|
||||||
It is therefore possible to have gaps in the data written to the target
|
It is therefore possible to have gaps in the data written to the target
|
||||||
files if restore fails or interrupted by the user.
|
files if restore fails or interrupted by the user.
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
Bugfix: Don't abort the stats command when data blobs are missing
|
Bugfix: Don't abort the stats command when data blobs are missing
|
||||||
|
|
||||||
Runing the stats command in the blobs-per-file mode on a repository with
|
Running the stats command in the blobs-per-file mode on a repository with
|
||||||
missing data blobs previously resulted in a crash.
|
missing data blobs previously resulted in a crash.
|
||||||
|
|
||||||
https://github.com/restic/restic/pull/2668
|
https://github.com/restic/restic/pull/2668
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
Bugfix: Mark repository files as read-only when using the local backend
|
Bugfix: Mark repository files as read-only when using the local backend
|
||||||
|
|
||||||
Files stored in a local repository were marked as writeable on the
|
Files stored in a local repository were marked as writable on the
|
||||||
filesystem for non-Windows systems, which did not prevent accidental file
|
filesystem for non-Windows systems, which did not prevent accidental file
|
||||||
modifications outside of restic. In addition, the local backend did not work
|
modifications outside of restic. In addition, the local backend did not work
|
||||||
with certain filesystems and network mounts which do not permit modifications
|
with certain filesystems and network mounts which do not permit modifications
|
||||||
|
|
|
@ -5,7 +5,7 @@ another process using an exclusive lock through a filesystem snapshot. Restic
|
||||||
was unable to backup those files before. This update enables backing up these
|
was unable to backup those files before. This update enables backing up these
|
||||||
files.
|
files.
|
||||||
|
|
||||||
This needs to be enabled explicitely using the --use-fs-snapshot option of the
|
This needs to be enabled explicitly using the --use-fs-snapshot option of the
|
||||||
backup command.
|
backup command.
|
||||||
|
|
||||||
https://github.com/restic/restic/issues/340
|
https://github.com/restic/restic/issues/340
|
||||||
|
|
|
@ -2,7 +2,7 @@ Enhancement: Parallelize scan of snapshot content in `copy` and `prune`
|
||||||
|
|
||||||
The `copy` and `prune` commands used to traverse the directories of
|
The `copy` and `prune` commands used to traverse the directories of
|
||||||
snapshots one by one to find used data. This snapshot traversal is
|
snapshots one by one to find used data. This snapshot traversal is
|
||||||
now parallized which can speed up this step several times.
|
now parallelized which can speed up this step several times.
|
||||||
|
|
||||||
In addition the `check` command now reports how many snapshots have
|
In addition the `check` command now reports how many snapshots have
|
||||||
already been processed.
|
already been processed.
|
||||||
|
|
|
@ -3,7 +3,7 @@ Enhancement: Add local metadata cache
|
||||||
We've added a local cache for metadata so that restic doesn't need to load
|
We've added a local cache for metadata so that restic doesn't need to load
|
||||||
all metadata (snapshots, indexes, ...) from the repo each time it starts. By
|
all metadata (snapshots, indexes, ...) from the repo each time it starts. By
|
||||||
default the cache is active, but there's a new global option `--no-cache`
|
default the cache is active, but there's a new global option `--no-cache`
|
||||||
that can be used to disable the cache. By deafult, the cache a standard
|
that can be used to disable the cache. By default, the cache a standard
|
||||||
cache folder for the OS, which can be overridden with `--cache-dir`. The
|
cache folder for the OS, which can be overridden with `--cache-dir`. The
|
||||||
cache will automatically populate, indexes and snapshots are saved as they
|
cache will automatically populate, indexes and snapshots are saved as they
|
||||||
are loaded. Cache directories for repos that haven't been used recently can
|
are loaded. Cache directories for repos that haven't been used recently can
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
Enhancement: Make `check` print `no errors found` explicitly
|
Enhancement: Make `check` print `no errors found` explicitly
|
||||||
|
|
||||||
The `check` command now explicetly prints `No errors were found` when no errors
|
The `check` command now explicitly prints `No errors were found` when no errors
|
||||||
could be found.
|
could be found.
|
||||||
|
|
||||||
https://github.com/restic/restic/pull/1319
|
https://github.com/restic/restic/pull/1319
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
Bugfix: Limit bandwith at the http.RoundTripper for HTTP based backends
|
Bugfix: Limit bandwidth at the http.RoundTripper for HTTP based backends
|
||||||
|
|
||||||
https://github.com/restic/restic/issues/1506
|
https://github.com/restic/restic/issues/1506
|
||||||
https://github.com/restic/restic/pull/1511
|
https://github.com/restic/restic/pull/1511
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
Bugfix: backup: Remove bandwidth display
|
Bugfix: backup: Remove bandwidth display
|
||||||
|
|
||||||
This commit removes the bandwidth displayed during backup process. It is
|
This commit removes the bandwidth displayed during backup process. It is
|
||||||
misleading and seldomly correct, because it's neither the "read
|
misleading and seldom correct, because it's neither the "read
|
||||||
bandwidth" (only for the very first backup) nor the "upload bandwidth".
|
bandwidth" (only for the very first backup) nor the "upload bandwidth".
|
||||||
Many users are confused about (and rightly so), c.f. #1581, #1033, #1591
|
Many users are confused about (and rightly so), c.f. #1581, #1033, #1591
|
||||||
|
|
||||||
|
|
|
@ -6,7 +6,7 @@ that means making a request (e.g. via HTTP) and returning an error when the
|
||||||
file already exists.
|
file already exists.
|
||||||
|
|
||||||
This is not accurate, the file could have been created between the HTTP request
|
This is not accurate, the file could have been created between the HTTP request
|
||||||
testing for it, and when writing starts, so we've relaxed this requeriment,
|
testing for it, and when writing starts, so we've relaxed this requirement,
|
||||||
which saves one additional HTTP request per newly added file.
|
which saves one additional HTTP request per newly added file.
|
||||||
|
|
||||||
https://github.com/restic/restic/pull/1623
|
https://github.com/restic/restic/pull/1623
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
Enhancement: Allow keeping a time range of snaphots
|
Enhancement: Allow keeping a time range of snapshots
|
||||||
|
|
||||||
We've added the `--keep-within` option to the `forget` command. It instructs
|
We've added the `--keep-within` option to the `forget` command. It instructs
|
||||||
restic to keep all snapshots within the given duration since the newest
|
restic to keep all snapshots within the given duration since the newest
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
Enhancement: Display reason why forget keeps snapshots
|
Enhancement: Display reason why forget keeps snapshots
|
||||||
|
|
||||||
We've added a column to the list of snapshots `forget` keeps which details the
|
We've added a column to the list of snapshots `forget` keeps which details the
|
||||||
reasons to keep a particuliar snapshot. This makes debugging policies for
|
reasons to keep a particular snapshot. This makes debugging policies for
|
||||||
forget much easier. Please remember to always try things out with `--dry-run`!
|
forget much easier. Please remember to always try things out with `--dry-run`!
|
||||||
|
|
||||||
https://github.com/restic/restic/pull/1876
|
https://github.com/restic/restic/pull/1876
|
||||||
|
|
|
@ -9,7 +9,7 @@ file should be noticed, and the modified file will be backed up. The ctime check
|
||||||
will be disabled if the --ignore-inode flag was given.
|
will be disabled if the --ignore-inode flag was given.
|
||||||
|
|
||||||
If this change causes problems for you, please open an issue, and we can look in
|
If this change causes problems for you, please open an issue, and we can look in
|
||||||
to adding a seperate flag to disable just the ctime check.
|
to adding a separate flag to disable just the ctime check.
|
||||||
|
|
||||||
https://github.com/restic/restic/issues/2179
|
https://github.com/restic/restic/issues/2179
|
||||||
https://github.com/restic/restic/pull/2212
|
https://github.com/restic/restic/pull/2212
|
||||||
|
|
|
@ -0,0 +1,19 @@
|
||||||
|
Enhancement: Support repositories with empty password
|
||||||
|
|
||||||
|
Restic refused to create or operate on repositories with an emtpy password.
|
||||||
|
Using the new option `--insecure-no-password` it is now possible to disable
|
||||||
|
this check. Restic will not prompt for a password when using this option.
|
||||||
|
For security reasons, the option must always be specified when operating on
|
||||||
|
repositories with an empty password.
|
||||||
|
|
||||||
|
Specifying `--insecure-no-password` while also passing a password to restic
|
||||||
|
via a CLI option or via environment variable results in an error.
|
||||||
|
|
||||||
|
The `init` and `copy` command also support the option `--from-insecure-no-password`
|
||||||
|
which applies to the source repository. The `key add` and `key passwd` comands
|
||||||
|
include the `--new-insecure-no-password` option to add or set an emtpy password.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/1786
|
||||||
|
https://github.com/restic/restic/issues/4326
|
||||||
|
https://github.com/restic/restic/pull/4698
|
||||||
|
https://github.com/restic/restic/pull/4808
|
|
@ -0,0 +1,11 @@
|
||||||
|
Bugfix: `backup` works if xattrs above the backup target cannot be read
|
||||||
|
|
||||||
|
When backup targets are specified using absolute paths, then `backup` also
|
||||||
|
includes information about the parent folders of the backup targets in the
|
||||||
|
snapshot. If the extended attributes for some of these folders could not be
|
||||||
|
read due to missing permissions, this caused the backup to fail. This has been
|
||||||
|
fixed.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/3600
|
||||||
|
https://github.com/restic/restic/pull/4668
|
||||||
|
https://forum.restic.net/t/parent-directories-above-the-snapshot-source-path-fatal-error-permission-denied/7216
|
|
@ -0,0 +1,11 @@
|
||||||
|
Enhancement: Make `prune` command resumable
|
||||||
|
|
||||||
|
When `prune` was interrupted, it a latter `prune` run previously started repacking
|
||||||
|
the pack files from the start as `prune` did not update the index while repacking.
|
||||||
|
|
||||||
|
The `prune` command now supports resuming interrupted prune runs. The update
|
||||||
|
of the repository index also has been optimized to use less memory and only
|
||||||
|
rewrite parts of the index that have changed.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/3806
|
||||||
|
https://github.com/restic/restic/pull/4812
|
|
@ -0,0 +1,7 @@
|
||||||
|
Bugfix: Fix slow sftp upload performance
|
||||||
|
|
||||||
|
Since restic 0.12.1, the upload speed of the sftp backend to a remote server
|
||||||
|
has regressed significantly. This has been fixed.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4209
|
||||||
|
https://github.com/restic/restic/pull/4782
|
|
@ -0,0 +1,16 @@
|
||||||
|
Enhancement: Support reading backup from a commands's standard output
|
||||||
|
|
||||||
|
The `backup` command now supports the `--stdin-from-command` option. When using
|
||||||
|
this option, the arguments to `backup` are interpreted as a command instead of
|
||||||
|
paths to back up. `backup` then executes the given command and stores the
|
||||||
|
standard output from it in the backup, similar to the what the `--stdin` option
|
||||||
|
does. This also enables restic to verify that the command completes with exit
|
||||||
|
code zero. A non-zero exit code causes the backup to fail.
|
||||||
|
|
||||||
|
Note that the `--stdin` option does not have to be specified at the same time,
|
||||||
|
and that the `--stdin-filename` option also applies to `--stdin-from-command`.
|
||||||
|
|
||||||
|
Example: `restic backup --stdin-from-command --stdin-filename dump.sql mysqldump [...]`
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4251
|
||||||
|
https://github.com/restic/restic/pull/4410
|
|
@ -0,0 +1,14 @@
|
||||||
|
Enhancement: support connection to rest-server using unix socket
|
||||||
|
|
||||||
|
Restic now supports connecting to rest-server using a unix socket for
|
||||||
|
rest-server version 0.13.0 or later.
|
||||||
|
|
||||||
|
This allows running restic as follows:
|
||||||
|
|
||||||
|
```
|
||||||
|
rest-server --listen unix:/tmp/rest.socket --data /path/to/data &
|
||||||
|
restic -r rest:http+unix:///tmp/rest.socket:/my_backup_repo/ [...]
|
||||||
|
```
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4287
|
||||||
|
https://github.com/restic/restic/pull/4655
|
|
@ -0,0 +1,9 @@
|
||||||
|
Enhancement: `check` command creates cache directory if it does not exist
|
||||||
|
|
||||||
|
If a custom cache directory was specified for the `check` command but the directory did not exist,
|
||||||
|
then `check` continued with cache disabled.
|
||||||
|
|
||||||
|
The `check` command now attempts to create the cache directory before initializing the cache.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4437
|
||||||
|
https://github.com/restic/restic/pull/4805
|
|
@ -0,0 +1,18 @@
|
||||||
|
Enhancement: Allow AWS Assume Role to be used for S3 backend
|
||||||
|
|
||||||
|
Previously only credentials discovered via the Minio discovery methods
|
||||||
|
were used to authenticate.
|
||||||
|
|
||||||
|
However, there are many circumstances where the discovered credentials have
|
||||||
|
lower permissions and need to assume a specific role. This is now possible
|
||||||
|
using the following new environment variables.
|
||||||
|
|
||||||
|
- RESTIC_AWS_ASSUME_ROLE_ARN
|
||||||
|
- RESTIC_AWS_ASSUME_ROLE_SESSION_NAME
|
||||||
|
- RESTIC_AWS_ASSUME_ROLE_EXTERNAL_ID
|
||||||
|
- RESTIC_AWS_ASSUME_ROLE_REGION (defaults to us-east-1)
|
||||||
|
- RESTIC_AWS_ASSUME_ROLE_POLICY
|
||||||
|
- RESTIC_AWS_ASSUME_ROLE_STS_ENDPOINT
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4472
|
||||||
|
https://github.com/restic/restic/pull/4474
|
|
@ -0,0 +1,6 @@
|
||||||
|
Change: Require at least ARMv6 for ARM binaries
|
||||||
|
|
||||||
|
The official release binaries of restic now require at least ARMv6 support for ARM platforms.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4540
|
||||||
|
https://github.com/restic/restic/pull/4542
|
|
@ -0,0 +1,7 @@
|
||||||
|
Enhancement: Add support for `--json` option to `version` command
|
||||||
|
|
||||||
|
Restic now supports outputting restic version and used go version, OS and
|
||||||
|
architecture via JSON when using the version command.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4547
|
||||||
|
https://github.com/restic/restic/pull/4553
|
|
@ -0,0 +1,11 @@
|
||||||
|
Enhancement: Add `--ncdu` option to `ls` command
|
||||||
|
|
||||||
|
NCDU (NCurses Disk Usage) is a tool to analyse disk usage of directories.
|
||||||
|
It has an option to save a directory tree and analyse it later.
|
||||||
|
The `ls` command now supports the `--ncdu` option which outputs information
|
||||||
|
about a snapshot in the NCDU format.
|
||||||
|
|
||||||
|
You can use it as follows: `restic ls latest --ncdu | ncdu -f -`
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4549
|
||||||
|
https://github.com/restic/restic/pull/4550
|
|
@ -0,0 +1,17 @@
|
||||||
|
Bugfix: Prevent `forget --keep-tags invalid` from deleting all snapshots
|
||||||
|
|
||||||
|
Running `forget --keep-tags invalid`, where the tag `invalid` does not
|
||||||
|
exist in the repository, would remove all snapshots. This is especially
|
||||||
|
problematic if the tag name contains a typo.
|
||||||
|
|
||||||
|
The `forget` command now fails with an error if all snapshots in a snapshot
|
||||||
|
group would be deleted. This prevents the above example from deleting all
|
||||||
|
snapshots.
|
||||||
|
|
||||||
|
It is possible to temporarily disable the new check by setting the environment variable
|
||||||
|
`RESTIC_FEATURES=safe-forget-keep-tags=false`. Note that this feature flag
|
||||||
|
will be removed in the next minor restic version.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4568
|
||||||
|
https://github.com/restic/restic/pull/4764
|
||||||
|
https://forum.restic.net/t/delete-all-snapshots-in-one-command-is-this-feature-intentional/6923/3
|
|
@ -0,0 +1,12 @@
|
||||||
|
Enhancement: Ignore s3.storage-class for metadata if archive tier is specified
|
||||||
|
|
||||||
|
There is no official cold storage support in restic, use this option at your
|
||||||
|
own risk.
|
||||||
|
|
||||||
|
Restic always stored all files on s3 using the specified `s3.storage-class`.
|
||||||
|
Now, restic will store metadata using a non-archive storage tier to avoid
|
||||||
|
problems when accessing a repository. To restore any data, it is still
|
||||||
|
necessary to manually warm up the required data beforehand.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4583
|
||||||
|
https://github.com/restic/restic/pull/4584
|
|
@ -0,0 +1,9 @@
|
||||||
|
Enhancement: Add support for feature flags
|
||||||
|
|
||||||
|
Restic now supports feature flags that can be used to enable and disable
|
||||||
|
experimental features. The flags can be set using the environment variable
|
||||||
|
`RESTIC_FEATURES`. To get a list of currently supported feature flags,
|
||||||
|
run the `features` command.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4601
|
||||||
|
https://github.com/restic/restic/pull/4666
|
|
@ -0,0 +1,23 @@
|
||||||
|
Change: Deprecate legacy index format and s3legacy layout
|
||||||
|
|
||||||
|
Support for the legacy index format used by restic before version 0.2.0 has
|
||||||
|
been deprecated and will be removed in the next minor restic version. You can
|
||||||
|
use `restic repair index` to update the index to the current format.
|
||||||
|
|
||||||
|
It is possible to temporarily reenable support for the legacy index format by
|
||||||
|
setting the environment variable
|
||||||
|
`RESTIC_FEATURES=deprecate-legacy-index=false`. Note that this feature flag
|
||||||
|
will be removed in the next minor restic version.
|
||||||
|
|
||||||
|
Support for the s3legacy layout used for the S3 backend before restic 0.7.0
|
||||||
|
has been deprecated and will be removed in the next minor restic version. You
|
||||||
|
can migrate your S3 repository using `RESTIC_FEATURES=deprecate-s3-legacy-layout=false restic migrate s3_layout`.
|
||||||
|
|
||||||
|
It is possible to temporarily reenable support for the legacy s3layout by
|
||||||
|
setting the environment variable
|
||||||
|
`RESTIC_FEATURES=deprecate-s3-legacy-layout=false`. Note that this feature flag
|
||||||
|
will be removed in the next minor restic version.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4602
|
||||||
|
https://github.com/restic/restic/pull/4724
|
||||||
|
https://github.com/restic/restic/pull/4743
|
|
@ -0,0 +1,31 @@
|
||||||
|
Change: Redesign backend error handling to improve reliability
|
||||||
|
|
||||||
|
Restic now downloads pack files in large chunks instead of using a streaming
|
||||||
|
download. This prevents failures due to interrupted streams. The `restore`
|
||||||
|
command now also retries downloading individual blobs that cannot be retrieved.
|
||||||
|
|
||||||
|
HTTP requests that are stuck for more than two minutes while uploading or
|
||||||
|
downloading are now forcibly interrupted. This ensures that stuck requests are
|
||||||
|
retried after a short timeout.
|
||||||
|
|
||||||
|
Attempts to access a missing file or a truncated file will no longer be retried.
|
||||||
|
This avoids unnecessary retries in those cases. All other backend requests are
|
||||||
|
retried for up to 15 minutes. This ensures that a temporarily interrupted network
|
||||||
|
connections can be tolerated.
|
||||||
|
|
||||||
|
If a download yields a corrupt file or blob, then the download will be retried once.
|
||||||
|
|
||||||
|
Most parts of the new backend error handling can temporarily be disabled by
|
||||||
|
setting the environment variable
|
||||||
|
`RESTIC_FEATURES=backend-error-redesign=false`. Note that this feature flag will
|
||||||
|
be removed in the next minor restic version.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4627
|
||||||
|
https://github.com/restic/restic/issues/4193
|
||||||
|
https://github.com/restic/restic/pull/4605
|
||||||
|
https://github.com/restic/restic/pull/4792
|
||||||
|
https://github.com/restic/restic/issues/4515
|
||||||
|
https://github.com/restic/restic/issues/1523
|
||||||
|
https://github.com/restic/restic/pull/4520
|
||||||
|
https://github.com/restic/restic/pull/4800
|
||||||
|
https://github.com/restic/restic/pull/4784
|
|
@ -0,0 +1,7 @@
|
||||||
|
Bugfix: Properly report the ID of newly added keys
|
||||||
|
|
||||||
|
`restic key add` now reports the ID of a newly added key. This simplifies
|
||||||
|
selecting a specific key using the `--key-hint key` option.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4656
|
||||||
|
https://github.com/restic/restic/pull/4657
|
|
@ -0,0 +1,8 @@
|
||||||
|
Enhancement: Move key add, list, remove and passwd as separate sub-commands
|
||||||
|
|
||||||
|
Restic now provides usage documentation for the `key` command. Each sub-command;
|
||||||
|
`add`, `list`, `remove` and `passwd` now have their own sub-command documentation
|
||||||
|
which can be invoked using `restic key <add|list|remove|passwd> --help`.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4676
|
||||||
|
https://github.com/restic/restic/pull/4685
|
|
@ -0,0 +1,8 @@
|
||||||
|
Enhancement: Add --target flag to the dump command
|
||||||
|
|
||||||
|
Restic `dump` always printed to the standard output. It now permits to select a
|
||||||
|
`--target` file to write the output to.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4678
|
||||||
|
https://github.com/restic/restic/pull/4682
|
||||||
|
https://github.com/restic/restic/pull/4692
|
|
@ -0,0 +1,10 @@
|
||||||
|
Enhancement: Allow specifying `--host` via environment variable
|
||||||
|
|
||||||
|
Restic commands that operate on snapshots, such as `restic backup` and
|
||||||
|
`restic snapshots`, support the `--host` flag to specify the hostname for
|
||||||
|
grouoping snapshots. They now permit selecting the hostname via the
|
||||||
|
environment variable `RESTIC_HOST`. `--host` still takes precedence over the
|
||||||
|
environment variable.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4733
|
||||||
|
https://github.com/restic/restic/pull/4734
|
|
@ -0,0 +1,9 @@
|
||||||
|
Change: Include full key ID in JSON output of `key list`
|
||||||
|
|
||||||
|
We have changed the JSON output of the `key list` command to include the full
|
||||||
|
key ID instead of just a shortened version, as the latter can be ambiguous
|
||||||
|
in some rare cases. To derive the short ID, please truncate the full ID down to
|
||||||
|
eight characters.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4744
|
||||||
|
https://github.com/restic/restic/pull/4745
|
|
@ -0,0 +1,8 @@
|
||||||
|
Bugfix: Fix possible error on concurrent cache cleanup
|
||||||
|
|
||||||
|
If multiple restic processes concurrently cleaned up no longer existing files
|
||||||
|
from the cache, this could cause some of the processes to fail with an `no such
|
||||||
|
file or directory` error. This has been fixed.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4760
|
||||||
|
https://github.com/restic/restic/pull/4761
|
|
@ -0,0 +1,8 @@
|
||||||
|
Enhancement: Allow custom User-Agent to be specified for outgoing requests
|
||||||
|
|
||||||
|
Restic now permits setting a custom `User-Agent` for outgoing HTTP requests
|
||||||
|
using the global flag `--http-user-agent` or the `RESTIC_HTTP_USER_AGENT`
|
||||||
|
environment variable.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/4768
|
||||||
|
https://github.com/restic/restic/pull/4810
|
|
@ -0,0 +1,11 @@
|
||||||
|
Enhancement: `backup` can omit snapshot creation if there was no change
|
||||||
|
|
||||||
|
The `backup` command always created a snapshot even if nothing changed
|
||||||
|
compared to the parent snapshot.
|
||||||
|
|
||||||
|
Restic now supports the `--skip-if-unchanged` option for the `backup`
|
||||||
|
command to omit creating a snapshot if the new snapshot's content would
|
||||||
|
be identical to that of the parent snapshot.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/662
|
||||||
|
https://github.com/restic/restic/pull/4816
|
|
@ -0,0 +1,12 @@
|
||||||
|
Enhancement: Support printing snapshot size in `snapshots` command
|
||||||
|
|
||||||
|
The `snapshots` command now supports printing the snapshot size for snapshots
|
||||||
|
created using this or a future restic version. For this, the `backup` command
|
||||||
|
now stores the backup summary statistics in the snapshot.
|
||||||
|
|
||||||
|
The text output of the `snapshots` command only shows the snapshot size. The
|
||||||
|
other statistics are only included in the JSON output. To inspect these
|
||||||
|
statistics use `restic snapshots --json` or `restic cat snapshot <snapshotID>`.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/693
|
||||||
|
https://github.com/restic/restic/pull/4705
|
|
@ -0,0 +1,11 @@
|
||||||
|
Enhancement: Improve `repair packs` command
|
||||||
|
|
||||||
|
The `repair packs` command has been improved to also be able to process
|
||||||
|
truncated pack files. The `check --read-data` command will provide instructions
|
||||||
|
on using the command if necessary to repair a repository. See the guide at
|
||||||
|
https://restic.readthedocs.io/en/stable/077_troubleshooting.html for further
|
||||||
|
instructions.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/828
|
||||||
|
https://github.com/restic/restic/pull/4644
|
||||||
|
https://github.com/restic/restic/pull/4655
|
|
@ -0,0 +1,22 @@
|
||||||
|
Enhancement: Add options to configure Windows Shadow Copy Service
|
||||||
|
|
||||||
|
Restic always used 120 seconds timeout and unconditionally created VSS snapshots
|
||||||
|
for all volume mount points on disk. Now this behavior can be fine-tuned by
|
||||||
|
new options, like exclude specific volumes and mount points or completely
|
||||||
|
disable auto snapshotting of volume mount points.
|
||||||
|
|
||||||
|
For example:
|
||||||
|
|
||||||
|
restic backup --use-fs-snapshot -o vss.timeout=5m -o vss.exclude-all-mount-points=true
|
||||||
|
|
||||||
|
changes timeout to five minutes and disable snapshotting of mount points on all volumes, and
|
||||||
|
|
||||||
|
restic backup --use-fs-snapshot -o vss.exclude-volumes="d:\;c:\mnt\;\\?\Volume{e2e0315d-9066-4f97-8343-eb5659b35762}"
|
||||||
|
|
||||||
|
excludes drive `d:`, mount point `c:\mnt` and specific volume from VSS snapshotting.
|
||||||
|
|
||||||
|
restic backup --use-fs-snapshot -o vss.provider={b5946137-7b9f-4925-af80-51abd60b20d5}
|
||||||
|
|
||||||
|
uses 'Microsoft Software Shadow Copy provider 1.0' instead of the default provider.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/pull/3067
|
|
@ -0,0 +1,16 @@
|
||||||
|
Enhancement: (alpha) Store deviceID only for hardlinks
|
||||||
|
|
||||||
|
Set `RESTIC_FEATURES=device-id-for-hardlinks` to enable this alpha feature.
|
||||||
|
The feature flag will be removed after repository format version 3 becomes
|
||||||
|
available or be replaced with a different solution.
|
||||||
|
|
||||||
|
When creating backups from a filesystem snapshot, for example created using
|
||||||
|
btrfs subvolumes, the deviceID of the filesystem changes compared to previous
|
||||||
|
snapshots. This prevented restic from deduplicating the directory metadata of
|
||||||
|
a snapshot.
|
||||||
|
|
||||||
|
When this alpha feature is enabled, then the deviceID is only stored for
|
||||||
|
hardlinks. This significantly reduces the metadata duplication for most
|
||||||
|
backups.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/pull/4006
|
|
@ -0,0 +1,7 @@
|
||||||
|
Enhancement: Significantly reduce prune memory usage
|
||||||
|
|
||||||
|
Prune has been optimized to use up to 60% less memory. The memory usage should
|
||||||
|
now be roughly similar to creating a backup.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/pull/4354
|
||||||
|
https://github.com/restic/restic/pull/4812
|
|
@ -0,0 +1,8 @@
|
||||||
|
Bugfix: Correct hardlink handling in `stats` command
|
||||||
|
|
||||||
|
If files on different devices had the same inode id, then the `stats` command
|
||||||
|
did not correctly calculate the snapshot size. This has been fixed.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/pull/4503
|
||||||
|
https://github.com/restic/restic/pull/4006
|
||||||
|
https://forum.restic.net/t/possible-bug-in-stats/6461/8
|
|
@ -0,0 +1,11 @@
|
||||||
|
Enhancement: Add bitrot detection to `diff` command
|
||||||
|
|
||||||
|
The output of the `diff` command now includes the modifier `?` for files
|
||||||
|
to indicate bitrot in backed up files. It will appear whenever there is a
|
||||||
|
difference in content while the metadata is exactly the same. Since files with
|
||||||
|
unchanged metadata are normally not read again when creating a backup, the
|
||||||
|
detection is only effective if the right-hand side of the diff has been created
|
||||||
|
with "backup --force".
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/805
|
||||||
|
https://github.com/restic/restic/pull/4526
|
|
@ -0,0 +1,5 @@
|
||||||
|
Enhancement: Add `--new-host` and `--new-time` options to `rewrite` command
|
||||||
|
|
||||||
|
`restic rewrite` now allows rewriting the host and / or time metadata of a snapshot.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/pull/4573
|
|
@ -0,0 +1,7 @@
|
||||||
|
Enhancement: `mount` tests mountpoint existence before opening the repository
|
||||||
|
|
||||||
|
The restic `mount` command now checks for the existence of the
|
||||||
|
mountpoint before opening the repository, leading to quicker error
|
||||||
|
detection.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/pull/4590
|
|
@ -0,0 +1,7 @@
|
||||||
|
Enhancement: Back up windows created time and file attributes like hidden flag
|
||||||
|
|
||||||
|
Restic did not back up windows-specific meta-data like created time and file attributes like hidden flag.
|
||||||
|
Restic now backs up file created time and file attributes like hidden, readonly and encrypted flag when backing up files and folders on Windows.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/pull/4611
|
||||||
|
|
|
@ -0,0 +1,6 @@
|
||||||
|
Bugfix: `find` ignored directories in some cases
|
||||||
|
|
||||||
|
In some cases, the `find` command ignored empty or moved directories. This has
|
||||||
|
been fixed.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/pull/4615
|
|
@ -0,0 +1,8 @@
|
||||||
|
Enhancement: `ls` uses `message_type` field to distinguish JSON messages
|
||||||
|
|
||||||
|
The `ls` command was the only command that used the `struct_type` field to determine
|
||||||
|
the message type in the JSON output format. Now, the JSON output of the
|
||||||
|
`ls` command also includes the `message_type`. The `struct_type` field is
|
||||||
|
still included, but it deprecated.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/pull/4664
|
|
@ -0,0 +1,9 @@
|
||||||
|
Bugfix: Shutdown cleanly when SIGTERM is received
|
||||||
|
|
||||||
|
Prior, if restic received SIGTERM it'd just immediately terminate skipping
|
||||||
|
cleanup- resulting in potential issues like stale locks being left behind.
|
||||||
|
|
||||||
|
This primarily effected containerized restic invocations- they use SIGTERM-
|
||||||
|
but this could be triggered via a simple `killall restic` in addition.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/pull/4703
|
|
@ -0,0 +1,11 @@
|
||||||
|
Enhancement: Back up and restore SecurityDescriptors on Windows
|
||||||
|
|
||||||
|
Restic now backs up and restores SecurityDescriptors when backing up files and folders
|
||||||
|
on Windows which includes owner, group, discretionary access control list (DACL),
|
||||||
|
system access control list (SACL). This requires the user to be a member of backup
|
||||||
|
operators or the application must be run as admin.
|
||||||
|
If that is not the case, only the current user's owner, group and DACL will be backed up
|
||||||
|
and during restore only the DACL of the backed file will be restored while the current
|
||||||
|
user's owner and group will be set during the restore.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/pull/4708
|
|
@ -0,0 +1,10 @@
|
||||||
|
Bugfix: Correct `--no-lock` handling of `ls` and `tag` command
|
||||||
|
|
||||||
|
The `ls` command never locked the repository. This has been fixed. The old
|
||||||
|
behavior is still supported using `ls --no-lock`. The latter invocation also
|
||||||
|
works with older restic versions.
|
||||||
|
|
||||||
|
The `tag` command erroneously accepted the `--no-lock` command. The command
|
||||||
|
now always requires an exclusive lock.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/pull/4709
|
|
@ -0,0 +1,5 @@
|
||||||
|
Enhancement: include snapshot id in reason field of forget JSON output
|
||||||
|
|
||||||
|
The JSON output of the `forget` command now includes the `id` and `short_id` of a snapshot in the `reason` field.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/pull/4737
|
|
@ -0,0 +1,8 @@
|
||||||
|
Enhancement: Remove all snapshots using `forget --unsafe-allow-remove-all`
|
||||||
|
|
||||||
|
The forget command now supports the `--unsafe-allow-remove-all` option. It must
|
||||||
|
always be combined with a snapshot filter (by host, path or tag).
|
||||||
|
For example the command `forget --tag example --unsafe-allow-remove-all`,
|
||||||
|
removes all snapshots with tag `example`.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/pull/4764
|
|
@ -0,0 +1,8 @@
|
||||||
|
Enhancement: Improve `dump` performance for large files
|
||||||
|
|
||||||
|
The `dump` command now retrieves the data chunks for a file in parallel. This
|
||||||
|
improves the download performance by up to the configured number of parallel
|
||||||
|
backend connections.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/issues/3406
|
||||||
|
https://github.com/restic/restic/pull/4796
|
|
@ -0,0 +1,5 @@
|
||||||
|
Enhancement: Add option to force use of Azure CLI credential
|
||||||
|
|
||||||
|
A new environment variable `AZURE_FORCE_CLI_CREDENTIAL=true` allows forcing the use of Azure CLI credential, ignoring other credentials like managed identity.
|
||||||
|
|
||||||
|
https://github.com/restic/restic/pull/4799
|
|
@ -1,89 +1,41 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"sync"
|
|
||||||
"syscall"
|
"syscall"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/debug"
|
"github.com/restic/restic/internal/debug"
|
||||||
)
|
)
|
||||||
|
|
||||||
var cleanupHandlers struct {
|
func createGlobalContext() context.Context {
|
||||||
sync.Mutex
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
list []func(code int) (int, error)
|
|
||||||
done bool
|
ch := make(chan os.Signal, 1)
|
||||||
ch chan os.Signal
|
go cleanupHandler(ch, cancel)
|
||||||
|
signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
|
||||||
|
|
||||||
|
return ctx
|
||||||
}
|
}
|
||||||
|
|
||||||
func init() {
|
// cleanupHandler handles the SIGINT and SIGTERM signals.
|
||||||
cleanupHandlers.ch = make(chan os.Signal, 1)
|
func cleanupHandler(c <-chan os.Signal, cancel context.CancelFunc) {
|
||||||
go CleanupHandler(cleanupHandlers.ch)
|
s := <-c
|
||||||
signal.Notify(cleanupHandlers.ch, syscall.SIGINT)
|
debug.Log("signal %v received, cleaning up", s)
|
||||||
}
|
Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s)
|
||||||
|
|
||||||
// AddCleanupHandler adds the function f to the list of cleanup handlers so
|
if val, _ := os.LookupEnv("RESTIC_DEBUG_STACKTRACE_SIGINT"); val != "" {
|
||||||
// that it is executed when all the cleanup handlers are run, e.g. when SIGINT
|
_, _ = os.Stderr.WriteString("\n--- STACKTRACE START ---\n\n")
|
||||||
// is received.
|
_, _ = os.Stderr.WriteString(debug.DumpStacktrace())
|
||||||
func AddCleanupHandler(f func(code int) (int, error)) {
|
_, _ = os.Stderr.WriteString("\n--- STACKTRACE END ---\n")
|
||||||
cleanupHandlers.Lock()
|
|
||||||
defer cleanupHandlers.Unlock()
|
|
||||||
|
|
||||||
// reset the done flag for integration tests
|
|
||||||
cleanupHandlers.done = false
|
|
||||||
|
|
||||||
cleanupHandlers.list = append(cleanupHandlers.list, f)
|
|
||||||
}
|
|
||||||
|
|
||||||
// RunCleanupHandlers runs all registered cleanup handlers
|
|
||||||
func RunCleanupHandlers(code int) int {
|
|
||||||
cleanupHandlers.Lock()
|
|
||||||
defer cleanupHandlers.Unlock()
|
|
||||||
|
|
||||||
if cleanupHandlers.done {
|
|
||||||
return code
|
|
||||||
}
|
}
|
||||||
cleanupHandlers.done = true
|
|
||||||
|
|
||||||
for _, f := range cleanupHandlers.list {
|
cancel()
|
||||||
var err error
|
|
||||||
code, err = f(code)
|
|
||||||
if err != nil {
|
|
||||||
Warnf("error in cleanup handler: %v\n", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
cleanupHandlers.list = nil
|
|
||||||
return code
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// CleanupHandler handles the SIGINT signals.
|
// Exit terminates the process with the given exit code.
|
||||||
func CleanupHandler(c <-chan os.Signal) {
|
|
||||||
for s := range c {
|
|
||||||
debug.Log("signal %v received, cleaning up", s)
|
|
||||||
Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s)
|
|
||||||
|
|
||||||
if val, _ := os.LookupEnv("RESTIC_DEBUG_STACKTRACE_SIGINT"); val != "" {
|
|
||||||
_, _ = os.Stderr.WriteString("\n--- STACKTRACE START ---\n\n")
|
|
||||||
_, _ = os.Stderr.WriteString(debug.DumpStacktrace())
|
|
||||||
_, _ = os.Stderr.WriteString("\n--- STACKTRACE END ---\n")
|
|
||||||
}
|
|
||||||
|
|
||||||
code := 0
|
|
||||||
|
|
||||||
if s == syscall.SIGINT {
|
|
||||||
code = 130
|
|
||||||
} else {
|
|
||||||
code = 1
|
|
||||||
}
|
|
||||||
|
|
||||||
Exit(code)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Exit runs the cleanup handlers and then terminates the process with the
|
|
||||||
// given exit code.
|
|
||||||
func Exit(code int) {
|
func Exit(code int) {
|
||||||
code = RunCleanupHandlers(code)
|
|
||||||
debug.Log("exiting with status code %d", code)
|
debug.Log("exiting with status code %d", code)
|
||||||
os.Exit(code)
|
os.Exit(code)
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,7 +12,6 @@ import (
|
||||||
"runtime"
|
"runtime"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
@ -25,7 +24,6 @@ import (
|
||||||
"github.com/restic/restic/internal/repository"
|
"github.com/restic/restic/internal/repository"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
"github.com/restic/restic/internal/textfile"
|
"github.com/restic/restic/internal/textfile"
|
||||||
"github.com/restic/restic/internal/ui"
|
|
||||||
"github.com/restic/restic/internal/ui/backup"
|
"github.com/restic/restic/internal/ui/backup"
|
||||||
"github.com/restic/restic/internal/ui/termstatus"
|
"github.com/restic/restic/internal/ui/termstatus"
|
||||||
)
|
)
|
||||||
|
@ -44,7 +42,7 @@ Exit status is 0 if the command was successful.
|
||||||
Exit status is 1 if there was a fatal error (no snapshot created).
|
Exit status is 1 if there was a fatal error (no snapshot created).
|
||||||
Exit status is 3 if some source data could not be read (incomplete snapshot created).
|
Exit status is 3 if some source data could not be read (incomplete snapshot created).
|
||||||
`,
|
`,
|
||||||
PreRun: func(cmd *cobra.Command, args []string) {
|
PreRun: func(_ *cobra.Command, _ []string) {
|
||||||
if backupOptions.Host == "" {
|
if backupOptions.Host == "" {
|
||||||
hostname, err := os.Hostname()
|
hostname, err := os.Hostname()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -56,31 +54,9 @@ Exit status is 3 if some source data could not be read (incomplete snapshot crea
|
||||||
},
|
},
|
||||||
DisableAutoGenTag: true,
|
DisableAutoGenTag: true,
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
ctx := cmd.Context()
|
term, cancel := setupTermstatus()
|
||||||
var wg sync.WaitGroup
|
defer cancel()
|
||||||
cancelCtx, cancel := context.WithCancel(ctx)
|
return runBackup(cmd.Context(), backupOptions, globalOptions, term, args)
|
||||||
defer func() {
|
|
||||||
// shutdown termstatus
|
|
||||||
cancel()
|
|
||||||
wg.Wait()
|
|
||||||
}()
|
|
||||||
|
|
||||||
term := termstatus.New(globalOptions.stdout, globalOptions.stderr, globalOptions.Quiet)
|
|
||||||
wg.Add(1)
|
|
||||||
go func() {
|
|
||||||
defer wg.Done()
|
|
||||||
term.Run(cancelCtx)
|
|
||||||
}()
|
|
||||||
|
|
||||||
// use the terminal for stdout/stderr
|
|
||||||
prevStdout, prevStderr := globalOptions.stdout, globalOptions.stderr
|
|
||||||
defer func() {
|
|
||||||
globalOptions.stdout, globalOptions.stderr = prevStdout, prevStderr
|
|
||||||
}()
|
|
||||||
stdioWrapper := ui.NewStdioWrapper(term)
|
|
||||||
globalOptions.stdout, globalOptions.stderr = stdioWrapper.Stdout(), stdioWrapper.Stderr()
|
|
||||||
|
|
||||||
return runBackup(ctx, backupOptions, globalOptions, term, args)
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -97,6 +73,7 @@ type BackupOptions struct {
|
||||||
ExcludeLargerThan string
|
ExcludeLargerThan string
|
||||||
Stdin bool
|
Stdin bool
|
||||||
StdinFilename string
|
StdinFilename string
|
||||||
|
StdinCommand bool
|
||||||
Tags restic.TagLists
|
Tags restic.TagLists
|
||||||
Host string
|
Host string
|
||||||
FilesFrom []string
|
FilesFrom []string
|
||||||
|
@ -110,6 +87,7 @@ type BackupOptions struct {
|
||||||
DryRun bool
|
DryRun bool
|
||||||
ReadConcurrency uint
|
ReadConcurrency uint
|
||||||
NoScan bool
|
NoScan bool
|
||||||
|
SkipIfUnchanged bool
|
||||||
}
|
}
|
||||||
|
|
||||||
var backupOptions BackupOptions
|
var backupOptions BackupOptions
|
||||||
|
@ -124,7 +102,7 @@ func init() {
|
||||||
f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time)")
|
f.StringVar(&backupOptions.Parent, "parent", "", "use this parent `snapshot` (default: latest snapshot in the group determined by --group-by and not newer than the timestamp determined by --time)")
|
||||||
backupOptions.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
|
backupOptions.GroupBy = restic.SnapshotGroupByOptions{Host: true, Path: true}
|
||||||
f.VarP(&backupOptions.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
|
f.VarP(&backupOptions.GroupBy, "group-by", "g", "`group` snapshots by host, paths and/or tags, separated by comma (disable grouping with '')")
|
||||||
f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories (overrides the "parent" flag)`)
|
f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the source files/directories (overrides the "parent" flag)`)
|
||||||
|
|
||||||
initExcludePatternOptions(f, &backupOptions.excludePatternOptions)
|
initExcludePatternOptions(f, &backupOptions.excludePatternOptions)
|
||||||
|
|
||||||
|
@ -134,9 +112,10 @@ func init() {
|
||||||
f.StringVar(&backupOptions.ExcludeLargerThan, "exclude-larger-than", "", "max `size` of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)")
|
f.StringVar(&backupOptions.ExcludeLargerThan, "exclude-larger-than", "", "max `size` of the files to be backed up (allowed suffixes: k/K, m/M, g/G, t/T)")
|
||||||
f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
|
f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
|
||||||
f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin")
|
f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "stdin", "`filename` to use when reading from stdin")
|
||||||
|
f.BoolVar(&backupOptions.StdinCommand, "stdin-from-command", false, "interpret arguments as command to execute and store its stdout")
|
||||||
f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)")
|
f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)")
|
||||||
f.UintVar(&backupOptions.ReadConcurrency, "read-concurrency", 0, "read `n` files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)")
|
f.UintVar(&backupOptions.ReadConcurrency, "read-concurrency", 0, "read `n` files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)")
|
||||||
f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
|
f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually (default: $RESTIC_HOST). To prevent an expensive rescan use the \"parent\" flag")
|
||||||
f.StringVar(&backupOptions.Host, "hostname", "", "set the `hostname` for the snapshot manually")
|
f.StringVar(&backupOptions.Host, "hostname", "", "set the `hostname` for the snapshot manually")
|
||||||
err := f.MarkDeprecated("hostname", "use --host")
|
err := f.MarkDeprecated("hostname", "use --host")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -148,17 +127,23 @@ func init() {
|
||||||
f.StringArrayVar(&backupOptions.FilesFromRaw, "files-from-raw", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
|
f.StringArrayVar(&backupOptions.FilesFromRaw, "files-from-raw", nil, "read the files to backup from `file` (can be combined with file args; can be specified multiple times)")
|
||||||
f.StringVar(&backupOptions.TimeStamp, "time", "", "`time` of the backup (ex. '2012-11-01 22:08:41') (default: now)")
|
f.StringVar(&backupOptions.TimeStamp, "time", "", "`time` of the backup (ex. '2012-11-01 22:08:41') (default: now)")
|
||||||
f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
|
f.BoolVar(&backupOptions.WithAtime, "with-atime", false, "store the atime for all files and directories")
|
||||||
f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number changes when checking for modified files")
|
f.BoolVar(&backupOptions.IgnoreInode, "ignore-inode", false, "ignore inode number and ctime changes when checking for modified files")
|
||||||
f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
|
f.BoolVar(&backupOptions.IgnoreCtime, "ignore-ctime", false, "ignore ctime changes when checking for modified files")
|
||||||
f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done")
|
f.BoolVarP(&backupOptions.DryRun, "dry-run", "n", false, "do not upload or write any data, just show what would be done")
|
||||||
f.BoolVar(&backupOptions.NoScan, "no-scan", false, "do not run scanner to estimate size of backup")
|
f.BoolVar(&backupOptions.NoScan, "no-scan", false, "do not run scanner to estimate size of backup")
|
||||||
if runtime.GOOS == "windows" {
|
if runtime.GOOS == "windows" {
|
||||||
f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
|
f.BoolVar(&backupOptions.UseFsSnapshot, "use-fs-snapshot", false, "use filesystem snapshot where possible (currently only Windows VSS)")
|
||||||
}
|
}
|
||||||
|
f.BoolVar(&backupOptions.SkipIfUnchanged, "skip-if-unchanged", false, "skip snapshot creation if identical to parent snapshot")
|
||||||
|
|
||||||
// parse read concurrency from env, on error the default value will be used
|
// parse read concurrency from env, on error the default value will be used
|
||||||
readConcurrency, _ := strconv.ParseUint(os.Getenv("RESTIC_READ_CONCURRENCY"), 10, 32)
|
readConcurrency, _ := strconv.ParseUint(os.Getenv("RESTIC_READ_CONCURRENCY"), 10, 32)
|
||||||
backupOptions.ReadConcurrency = uint(readConcurrency)
|
backupOptions.ReadConcurrency = uint(readConcurrency)
|
||||||
|
|
||||||
|
// parse host from env, if not exists or empty the default value will be used
|
||||||
|
if host := os.Getenv("RESTIC_HOST"); host != "" {
|
||||||
|
backupOptions.Host = host
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// filterExisting returns a slice of all existing items, or an error if no
|
// filterExisting returns a slice of all existing items, or an error if no
|
||||||
|
@ -175,7 +160,7 @@ func filterExisting(items []string) (result []string, err error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(result) == 0 {
|
if len(result) == 0 {
|
||||||
return nil, errors.Fatal("all target directories/files do not exist")
|
return nil, errors.Fatal("all source directories/files do not exist")
|
||||||
}
|
}
|
||||||
|
|
||||||
return
|
return
|
||||||
|
@ -274,7 +259,7 @@ func readFilenamesRaw(r io.Reader) (names []string, err error) {
|
||||||
|
|
||||||
// Check returns an error when an invalid combination of options was set.
|
// Check returns an error when an invalid combination of options was set.
|
||||||
func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
|
func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
|
||||||
if gopts.password == "" {
|
if gopts.password == "" && !gopts.InsecureNoPassword {
|
||||||
if opts.Stdin {
|
if opts.Stdin {
|
||||||
return errors.Fatal("cannot read both password and data from stdin")
|
return errors.Fatal("cannot read both password and data from stdin")
|
||||||
}
|
}
|
||||||
|
@ -287,7 +272,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if opts.Stdin {
|
if opts.Stdin || opts.StdinCommand {
|
||||||
if len(opts.FilesFrom) > 0 {
|
if len(opts.FilesFrom) > 0 {
|
||||||
return errors.Fatal("--stdin and --files-from cannot be used together")
|
return errors.Fatal("--stdin and --files-from cannot be used together")
|
||||||
}
|
}
|
||||||
|
@ -298,7 +283,7 @@ func (opts BackupOptions) Check(gopts GlobalOptions, args []string) error {
|
||||||
return errors.Fatal("--stdin and --files-from-raw cannot be used together")
|
return errors.Fatal("--stdin and --files-from-raw cannot be used together")
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(args) > 0 {
|
if len(args) > 0 && !opts.StdinCommand {
|
||||||
return errors.Fatal("--stdin was specified and files/dirs were listed as arguments")
|
return errors.Fatal("--stdin was specified and files/dirs were listed as arguments")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -366,7 +351,7 @@ func collectRejectFuncs(opts BackupOptions, targets []string) (fs []RejectFunc,
|
||||||
|
|
||||||
// collectTargets returns a list of target files/dirs from several sources.
|
// collectTargets returns a list of target files/dirs from several sources.
|
||||||
func collectTargets(opts BackupOptions, args []string) (targets []string, err error) {
|
func collectTargets(opts BackupOptions, args []string) (targets []string, err error) {
|
||||||
if opts.Stdin {
|
if opts.Stdin || opts.StdinCommand {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -420,7 +405,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
|
||||||
// and have the ability to use both files-from and args at the same time.
|
// and have the ability to use both files-from and args at the same time.
|
||||||
targets = append(targets, args...)
|
targets = append(targets, args...)
|
||||||
if len(targets) == 0 && !opts.Stdin {
|
if len(targets) == 0 && !opts.Stdin {
|
||||||
return nil, errors.Fatal("nothing to backup, please specify target files/dirs")
|
return nil, errors.Fatal("nothing to backup, please specify source files/dirs")
|
||||||
}
|
}
|
||||||
|
|
||||||
targets, err = filterExisting(targets)
|
targets, err = filterExisting(targets)
|
||||||
|
@ -433,7 +418,7 @@ func collectTargets(opts BackupOptions, args []string) (targets []string, err er
|
||||||
|
|
||||||
// parent returns the ID of the parent snapshot. If there is none, nil is
|
// parent returns the ID of the parent snapshot. If there is none, nil is
|
||||||
// returned.
|
// returned.
|
||||||
func findParentSnapshot(ctx context.Context, repo restic.Repository, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) {
|
func findParentSnapshot(ctx context.Context, repo restic.ListerLoaderUnpacked, opts BackupOptions, targets []string, timeStampLimit time.Time) (*restic.Snapshot, error) {
|
||||||
if opts.Force {
|
if opts.Force {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
@ -453,7 +438,7 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
|
||||||
f.Tags = []restic.TagList{opts.Tags.Flatten()}
|
f.Tags = []restic.TagList{opts.Tags.Flatten()}
|
||||||
}
|
}
|
||||||
|
|
||||||
sn, _, err := f.FindLatest(ctx, repo.Backend(), repo, snName)
|
sn, _, err := f.FindLatest(ctx, repo, repo, snName)
|
||||||
// Snapshot not found is ok if no explicit parent was set
|
// Snapshot not found is ok if no explicit parent was set
|
||||||
if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) {
|
if opts.Parent == "" && errors.Is(err, restic.ErrNoSnapshotFound) {
|
||||||
err = nil
|
err = nil
|
||||||
|
@ -462,7 +447,16 @@ func findParentSnapshot(ctx context.Context, repo restic.Repository, opts Backup
|
||||||
}
|
}
|
||||||
|
|
||||||
func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
|
func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
|
||||||
err := opts.Check(gopts, args)
|
var vsscfg fs.VSSConfig
|
||||||
|
var err error
|
||||||
|
|
||||||
|
if runtime.GOOS == "windows" {
|
||||||
|
if vsscfg, err = fs.ParseVSSConfig(gopts.extended); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
err = opts.Check(gopts, args)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -473,6 +467,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
|
||||||
}
|
}
|
||||||
|
|
||||||
timeStamp := time.Now()
|
timeStamp := time.Now()
|
||||||
|
backupStart := timeStamp
|
||||||
if opts.TimeStamp != "" {
|
if opts.TimeStamp != "" {
|
||||||
timeStamp, err = time.ParseInLocation(TimeFormat, opts.TimeStamp, time.Local)
|
timeStamp, err = time.ParseInLocation(TimeFormat, opts.TimeStamp, time.Local)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -484,10 +479,11 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
|
||||||
Verbosef("open repository\n")
|
Verbosef("open repository\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
repo, err := OpenRepository(ctx, gopts)
|
ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, opts.DryRun)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
defer unlock()
|
||||||
|
|
||||||
var progressPrinter backup.ProgressPrinter
|
var progressPrinter backup.ProgressPrinter
|
||||||
if gopts.JSON {
|
if gopts.JSON {
|
||||||
|
@ -499,22 +495,6 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
|
||||||
calculateProgressInterval(!gopts.Quiet, gopts.JSON))
|
calculateProgressInterval(!gopts.Quiet, gopts.JSON))
|
||||||
defer progressReporter.Done()
|
defer progressReporter.Done()
|
||||||
|
|
||||||
if opts.DryRun {
|
|
||||||
repo.SetDryRun()
|
|
||||||
}
|
|
||||||
|
|
||||||
if !gopts.JSON {
|
|
||||||
progressPrinter.V("lock repository")
|
|
||||||
}
|
|
||||||
if !opts.DryRun {
|
|
||||||
var lock *restic.Lock
|
|
||||||
lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
|
|
||||||
defer unlockRepo(lock)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// rejectByNameFuncs collect functions that can reject items from the backup based on path only
|
// rejectByNameFuncs collect functions that can reject items from the backup based on path only
|
||||||
rejectByNameFuncs, err := collectRejectByNameFuncs(opts, repo)
|
rejectByNameFuncs, err := collectRejectByNameFuncs(opts, repo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -578,8 +558,8 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
errorHandler := func(item string, err error) error {
|
errorHandler := func(item string, err error) {
|
||||||
return progressReporter.Error(item, err)
|
_ = progressReporter.Error(item, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
messageHandler := func(msg string, args ...interface{}) {
|
messageHandler := func(msg string, args ...interface{}) {
|
||||||
|
@ -588,20 +568,28 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
localVss := fs.NewLocalVss(errorHandler, messageHandler)
|
localVss := fs.NewLocalVss(errorHandler, messageHandler, vsscfg)
|
||||||
defer localVss.DeleteSnapshots()
|
defer localVss.DeleteSnapshots()
|
||||||
targetFS = localVss
|
targetFS = localVss
|
||||||
}
|
}
|
||||||
if opts.Stdin {
|
|
||||||
|
if opts.Stdin || opts.StdinCommand {
|
||||||
if !gopts.JSON {
|
if !gopts.JSON {
|
||||||
progressPrinter.V("read data from stdin")
|
progressPrinter.V("read data from stdin")
|
||||||
}
|
}
|
||||||
filename := path.Join("/", opts.StdinFilename)
|
filename := path.Join("/", opts.StdinFilename)
|
||||||
|
var source io.ReadCloser = os.Stdin
|
||||||
|
if opts.StdinCommand {
|
||||||
|
source, err = fs.NewCommandReader(ctx, args, globalOptions.stderr)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
targetFS = &fs.Reader{
|
targetFS = &fs.Reader{
|
||||||
ModTime: timeStamp,
|
ModTime: timeStamp,
|
||||||
Name: filename,
|
Name: filename,
|
||||||
Mode: 0644,
|
Mode: 0644,
|
||||||
ReadCloser: os.Stdin,
|
ReadCloser: source,
|
||||||
}
|
}
|
||||||
targets = []string{filename}
|
targets = []string{filename}
|
||||||
}
|
}
|
||||||
|
@ -623,14 +611,20 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
|
||||||
wg.Go(func() error { return sc.Scan(cancelCtx, targets) })
|
wg.Go(func() error { return sc.Scan(cancelCtx, targets) })
|
||||||
}
|
}
|
||||||
|
|
||||||
arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: backupOptions.ReadConcurrency})
|
arch := archiver.New(repo, targetFS, archiver.Options{ReadConcurrency: opts.ReadConcurrency})
|
||||||
arch.SelectByName = selectByNameFilter
|
arch.SelectByName = selectByNameFilter
|
||||||
arch.Select = selectFilter
|
arch.Select = selectFilter
|
||||||
arch.WithAtime = opts.WithAtime
|
arch.WithAtime = opts.WithAtime
|
||||||
success := true
|
success := true
|
||||||
arch.Error = func(item string, err error) error {
|
arch.Error = func(item string, err error) error {
|
||||||
success = false
|
success = false
|
||||||
return progressReporter.Error(item, err)
|
reterr := progressReporter.Error(item, err)
|
||||||
|
// If we receive a fatal error during the execution of the snapshot,
|
||||||
|
// we abort the snapshot.
|
||||||
|
if reterr == nil && errors.IsFatal(err) {
|
||||||
|
reterr = err
|
||||||
|
}
|
||||||
|
return reterr
|
||||||
}
|
}
|
||||||
arch.CompleteItem = progressReporter.CompleteItem
|
arch.CompleteItem = progressReporter.CompleteItem
|
||||||
arch.StartFile = progressReporter.StartFile
|
arch.StartFile = progressReporter.StartFile
|
||||||
|
@ -646,18 +640,20 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
|
||||||
}
|
}
|
||||||
|
|
||||||
snapshotOpts := archiver.SnapshotOptions{
|
snapshotOpts := archiver.SnapshotOptions{
|
||||||
Excludes: opts.Excludes,
|
Excludes: opts.Excludes,
|
||||||
Tags: opts.Tags.Flatten(),
|
Tags: opts.Tags.Flatten(),
|
||||||
Time: timeStamp,
|
BackupStart: backupStart,
|
||||||
Hostname: opts.Host,
|
Time: timeStamp,
|
||||||
ParentSnapshot: parentSnapshot,
|
Hostname: opts.Host,
|
||||||
ProgramVersion: "restic " + version,
|
ParentSnapshot: parentSnapshot,
|
||||||
|
ProgramVersion: "restic " + version,
|
||||||
|
SkipIfUnchanged: opts.SkipIfUnchanged,
|
||||||
}
|
}
|
||||||
|
|
||||||
if !gopts.JSON {
|
if !gopts.JSON {
|
||||||
progressPrinter.V("start backup on %v", targets)
|
progressPrinter.V("start backup on %v", targets)
|
||||||
}
|
}
|
||||||
_, id, err := arch.Snapshot(ctx, targets, snapshotOpts)
|
_, id, summary, err := arch.Snapshot(ctx, targets, snapshotOpts)
|
||||||
|
|
||||||
// cleanly shutdown all running goroutines
|
// cleanly shutdown all running goroutines
|
||||||
cancel()
|
cancel()
|
||||||
|
@ -671,10 +667,7 @@ func runBackup(ctx context.Context, opts BackupOptions, gopts GlobalOptions, ter
|
||||||
}
|
}
|
||||||
|
|
||||||
// Report finished execution
|
// Report finished execution
|
||||||
progressReporter.Finish(id, opts.DryRun)
|
progressReporter.Finish(id, summary, opts.DryRun)
|
||||||
if !gopts.JSON && !opts.DryRun {
|
|
||||||
progressPrinter.P("snapshot %s saved\n", id.Str())
|
|
||||||
}
|
|
||||||
if !success {
|
if !success {
|
||||||
return ErrInvalidSourceData
|
return ErrInvalidSourceData
|
||||||
}
|
}
|
||||||
|
|
|
@ -249,29 +249,18 @@ func TestBackupTreeLoadError(t *testing.T) {
|
||||||
opts := BackupOptions{}
|
opts := BackupOptions{}
|
||||||
// Backup a subdirectory first, such that we can remove the tree pack for the subdirectory
|
// Backup a subdirectory first, such that we can remove the tree pack for the subdirectory
|
||||||
testRunBackup(t, env.testdata, []string{"test"}, opts, env.gopts)
|
testRunBackup(t, env.testdata, []string{"test"}, opts, env.gopts)
|
||||||
|
treePacks := listTreePacks(env.gopts, t)
|
||||||
r, err := OpenRepository(context.TODO(), env.gopts)
|
|
||||||
rtest.OK(t, err)
|
|
||||||
rtest.OK(t, r.LoadIndex(context.TODO(), nil))
|
|
||||||
treePacks := restic.NewIDSet()
|
|
||||||
r.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
|
|
||||||
if pb.Type == restic.TreeBlob {
|
|
||||||
treePacks.Insert(pb.PackID)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
|
testRunBackup(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
|
||||||
testRunCheck(t, env.gopts)
|
testRunCheck(t, env.gopts)
|
||||||
|
|
||||||
// delete the subdirectory pack first
|
// delete the subdirectory pack first
|
||||||
for id := range treePacks {
|
removePacks(env.gopts, t, treePacks)
|
||||||
rtest.OK(t, r.Backend().Remove(context.TODO(), restic.Handle{Type: restic.PackFile, Name: id.String()}))
|
|
||||||
}
|
|
||||||
testRunRebuildIndex(t, env.gopts)
|
testRunRebuildIndex(t, env.gopts)
|
||||||
// now the repo is missing the tree blob in the index; check should report this
|
// now the repo is missing the tree blob in the index; check should report this
|
||||||
testRunCheckMustFail(t, env.gopts)
|
testRunCheckMustFail(t, env.gopts)
|
||||||
// second backup should report an error but "heal" this situation
|
// second backup should report an error but "heal" this situation
|
||||||
err = testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
|
err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{filepath.Base(env.testdata)}, opts, env.gopts)
|
||||||
rtest.Assert(t, err != nil, "backup should have reported an error for the subdirectory")
|
rtest.Assert(t, err != nil, "backup should have reported an error for the subdirectory")
|
||||||
testRunCheck(t, env.gopts)
|
testRunCheck(t, env.gopts)
|
||||||
|
|
||||||
|
@ -405,6 +394,7 @@ func TestIncrementalBackup(t *testing.T) {
|
||||||
t.Logf("repository grown by %d bytes", stat3.size-stat2.size)
|
t.Logf("repository grown by %d bytes", stat3.size-stat2.size)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// nolint: staticcheck // false positive nil pointer dereference check
|
||||||
func TestBackupTags(t *testing.T) {
|
func TestBackupTags(t *testing.T) {
|
||||||
env, cleanup := withTestEnvironment(t)
|
env, cleanup := withTestEnvironment(t)
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
@ -440,6 +430,7 @@ func TestBackupTags(t *testing.T) {
|
||||||
"expected parent to be %v, got %v", parent.ID, newest.Parent)
|
"expected parent to be %v, got %v", parent.ID, newest.Parent)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// nolint: staticcheck // false positive nil pointer dereference check
|
||||||
func TestBackupProgramVersion(t *testing.T) {
|
func TestBackupProgramVersion(t *testing.T) {
|
||||||
env, cleanup := withTestEnvironment(t)
|
env, cleanup := withTestEnvironment(t)
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
@ -567,3 +558,101 @@ func linkEqual(source, dest []string) bool {
|
||||||
|
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStdinFromCommand(t *testing.T) {
|
||||||
|
env, cleanup := withTestEnvironment(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
testSetupBackupData(t, env)
|
||||||
|
opts := BackupOptions{
|
||||||
|
StdinCommand: true,
|
||||||
|
StdinFilename: "stdin",
|
||||||
|
}
|
||||||
|
|
||||||
|
testRunBackup(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; print('something'); sys.exit(0)"}, opts, env.gopts)
|
||||||
|
testListSnapshots(t, env.gopts, 1)
|
||||||
|
|
||||||
|
testRunCheck(t, env.gopts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStdinFromCommandNoOutput(t *testing.T) {
|
||||||
|
env, cleanup := withTestEnvironment(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
testSetupBackupData(t, env)
|
||||||
|
opts := BackupOptions{
|
||||||
|
StdinCommand: true,
|
||||||
|
StdinFilename: "stdin",
|
||||||
|
}
|
||||||
|
|
||||||
|
err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; sys.exit(0)"}, opts, env.gopts)
|
||||||
|
rtest.Assert(t, err != nil && err.Error() == "at least one source file could not be read", "No data error expected")
|
||||||
|
testListSnapshots(t, env.gopts, 1)
|
||||||
|
|
||||||
|
testRunCheck(t, env.gopts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStdinFromCommandFailExitCode(t *testing.T) {
|
||||||
|
env, cleanup := withTestEnvironment(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
testSetupBackupData(t, env)
|
||||||
|
opts := BackupOptions{
|
||||||
|
StdinCommand: true,
|
||||||
|
StdinFilename: "stdin",
|
||||||
|
}
|
||||||
|
|
||||||
|
err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; print('test'); sys.exit(1)"}, opts, env.gopts)
|
||||||
|
rtest.Assert(t, err != nil, "Expected error while backing up")
|
||||||
|
|
||||||
|
testListSnapshots(t, env.gopts, 0)
|
||||||
|
|
||||||
|
testRunCheck(t, env.gopts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestStdinFromCommandFailNoOutputAndExitCode(t *testing.T) {
|
||||||
|
env, cleanup := withTestEnvironment(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
testSetupBackupData(t, env)
|
||||||
|
opts := BackupOptions{
|
||||||
|
StdinCommand: true,
|
||||||
|
StdinFilename: "stdin",
|
||||||
|
}
|
||||||
|
|
||||||
|
err := testRunBackupAssumeFailure(t, filepath.Dir(env.testdata), []string{"python", "-c", "import sys; sys.exit(1)"}, opts, env.gopts)
|
||||||
|
rtest.Assert(t, err != nil, "Expected error while backing up")
|
||||||
|
|
||||||
|
testListSnapshots(t, env.gopts, 0)
|
||||||
|
|
||||||
|
testRunCheck(t, env.gopts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackupEmptyPassword(t *testing.T) {
|
||||||
|
// basic sanity test that empty passwords work
|
||||||
|
env, cleanup := withTestEnvironment(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
env.gopts.password = ""
|
||||||
|
env.gopts.InsecureNoPassword = true
|
||||||
|
|
||||||
|
testSetupBackupData(t, env)
|
||||||
|
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, BackupOptions{}, env.gopts)
|
||||||
|
testListSnapshots(t, env.gopts, 1)
|
||||||
|
testRunCheck(t, env.gopts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackupSkipIfUnchanged(t *testing.T) {
|
||||||
|
env, cleanup := withTestEnvironment(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
testSetupBackupData(t, env)
|
||||||
|
opts := BackupOptions{SkipIfUnchanged: true}
|
||||||
|
|
||||||
|
for i := 0; i < 3; i++ {
|
||||||
|
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
|
||||||
|
testListSnapshots(t, env.gopts, 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
testRunCheck(t, env.gopts)
|
||||||
|
}
|
||||||
|
|
|
@ -8,7 +8,7 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/cache"
|
"github.com/restic/restic/internal/backend/cache"
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
"github.com/restic/restic/internal/fs"
|
"github.com/restic/restic/internal/fs"
|
||||||
"github.com/restic/restic/internal/ui"
|
"github.com/restic/restic/internal/ui"
|
||||||
|
@ -28,7 +28,7 @@ EXIT STATUS
|
||||||
Exit status is 0 if the command was successful, and non-zero if there was any error.
|
Exit status is 0 if the command was successful, and non-zero if there was any error.
|
||||||
`,
|
`,
|
||||||
DisableAutoGenTag: true,
|
DisableAutoGenTag: true,
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(_ *cobra.Command, args []string) error {
|
||||||
return runCache(cacheOptions, globalOptions, args)
|
return runCache(cacheOptions, globalOptions, args)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,7 +7,6 @@ import (
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/backend"
|
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
"github.com/restic/restic/internal/repository"
|
"github.com/restic/restic/internal/repository"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
|
@ -64,19 +63,11 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
repo, err := OpenRepository(ctx, gopts)
|
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
defer unlock()
|
||||||
if !gopts.NoLock {
|
|
||||||
var lock *restic.Lock
|
|
||||||
lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
|
|
||||||
defer unlockRepo(lock)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
tpe := args[0]
|
tpe := args[0]
|
||||||
|
|
||||||
|
@ -106,7 +97,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
|
||||||
Println(string(buf))
|
Println(string(buf))
|
||||||
return nil
|
return nil
|
||||||
case "snapshot":
|
case "snapshot":
|
||||||
sn, _, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
|
sn, _, err := restic.FindSnapshot(ctx, repo, repo, args[1])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Fatalf("could not find snapshot: %v\n", err)
|
return errors.Fatalf("could not find snapshot: %v\n", err)
|
||||||
}
|
}
|
||||||
|
@ -154,9 +145,9 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
|
||||||
return nil
|
return nil
|
||||||
|
|
||||||
case "pack":
|
case "pack":
|
||||||
h := restic.Handle{Type: restic.PackFile, Name: id.String()}
|
buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
|
||||||
buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
|
// allow returning broken pack files
|
||||||
if err != nil {
|
if buf == nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -176,8 +167,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, t := range []restic.BlobType{restic.DataBlob, restic.TreeBlob} {
|
for _, t := range []restic.BlobType{restic.DataBlob, restic.TreeBlob} {
|
||||||
bh := restic.BlobHandle{ID: id, Type: t}
|
if _, ok := repo.LookupBlobSize(t, id); !ok {
|
||||||
if !repo.Index().Has(bh) {
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -193,7 +183,7 @@ func runCat(ctx context.Context, gopts GlobalOptions, args []string) error {
|
||||||
return errors.Fatal("blob not found")
|
return errors.Fatal("blob not found")
|
||||||
|
|
||||||
case "tree":
|
case "tree":
|
||||||
sn, subfolder, err := restic.FindSnapshot(ctx, repo.Backend(), repo, args[1])
|
sn, subfolder, err := restic.FindSnapshot(ctx, repo, repo, args[1])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Fatalf("could not find snapshot: %v\n", err)
|
return errors.Fatalf("could not find snapshot: %v\n", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,12 +11,15 @@ import (
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/cache"
|
"github.com/restic/restic/internal/backend/cache"
|
||||||
"github.com/restic/restic/internal/checker"
|
"github.com/restic/restic/internal/checker"
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
"github.com/restic/restic/internal/fs"
|
"github.com/restic/restic/internal/fs"
|
||||||
|
"github.com/restic/restic/internal/repository"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
"github.com/restic/restic/internal/ui"
|
"github.com/restic/restic/internal/ui"
|
||||||
|
"github.com/restic/restic/internal/ui/progress"
|
||||||
|
"github.com/restic/restic/internal/ui/termstatus"
|
||||||
)
|
)
|
||||||
|
|
||||||
var cmdCheck = &cobra.Command{
|
var cmdCheck = &cobra.Command{
|
||||||
|
@ -36,9 +39,11 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
|
||||||
`,
|
`,
|
||||||
DisableAutoGenTag: true,
|
DisableAutoGenTag: true,
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
return runCheck(cmd.Context(), checkOptions, globalOptions, args)
|
term, cancel := setupTermstatus()
|
||||||
|
defer cancel()
|
||||||
|
return runCheck(cmd.Context(), checkOptions, globalOptions, args, term)
|
||||||
},
|
},
|
||||||
PreRunE: func(cmd *cobra.Command, args []string) error {
|
PreRunE: func(_ *cobra.Command, _ []string) error {
|
||||||
return checkFlags(checkOptions)
|
return checkFlags(checkOptions)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
@ -154,7 +159,7 @@ func parsePercentage(s string) (float64, error) {
|
||||||
// - if the user explicitly requested --no-cache, we don't use any cache
|
// - if the user explicitly requested --no-cache, we don't use any cache
|
||||||
// - if the user provides --cache-dir, we use a cache in a temporary sub-directory of the specified directory and the sub-directory is deleted after the check
|
// - if the user provides --cache-dir, we use a cache in a temporary sub-directory of the specified directory and the sub-directory is deleted after the check
|
||||||
// - by default, we use a cache in a temporary directory that is deleted after the check
|
// - by default, we use a cache in a temporary directory that is deleted after the check
|
||||||
func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func()) {
|
func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions, printer progress.Printer) (cleanup func()) {
|
||||||
cleanup = func() {}
|
cleanup = func() {}
|
||||||
if opts.WithCache {
|
if opts.WithCache {
|
||||||
// use the default cache, no setup needed
|
// use the default cache, no setup needed
|
||||||
|
@ -172,52 +177,51 @@ func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func())
|
||||||
}
|
}
|
||||||
|
|
||||||
// use a cache in a temporary directory
|
// use a cache in a temporary directory
|
||||||
|
err := os.MkdirAll(cachedir, 0755)
|
||||||
|
if err != nil {
|
||||||
|
Warnf("unable to create cache directory %s, disabling cache: %v\n", cachedir, err)
|
||||||
|
gopts.NoCache = true
|
||||||
|
return cleanup
|
||||||
|
}
|
||||||
tempdir, err := os.MkdirTemp(cachedir, "restic-check-cache-")
|
tempdir, err := os.MkdirTemp(cachedir, "restic-check-cache-")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// if an error occurs, don't use any cache
|
// if an error occurs, don't use any cache
|
||||||
Warnf("unable to create temporary directory for cache during check, disabling cache: %v\n", err)
|
printer.E("unable to create temporary directory for cache during check, disabling cache: %v\n", err)
|
||||||
gopts.NoCache = true
|
gopts.NoCache = true
|
||||||
return cleanup
|
return cleanup
|
||||||
}
|
}
|
||||||
|
|
||||||
gopts.CacheDir = tempdir
|
gopts.CacheDir = tempdir
|
||||||
Verbosef("using temporary cache in %v\n", tempdir)
|
printer.P("using temporary cache in %v\n", tempdir)
|
||||||
|
|
||||||
cleanup = func() {
|
cleanup = func() {
|
||||||
err := fs.RemoveAll(tempdir)
|
err := fs.RemoveAll(tempdir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Warnf("error removing temporary cache directory: %v\n", err)
|
printer.E("error removing temporary cache directory: %v\n", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return cleanup
|
return cleanup
|
||||||
}
|
}
|
||||||
|
|
||||||
func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string) error {
|
func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args []string, term *termstatus.Terminal) error {
|
||||||
if len(args) != 0 {
|
if len(args) != 0 {
|
||||||
return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
|
return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
|
||||||
}
|
}
|
||||||
|
|
||||||
cleanup := prepareCheckCache(opts, &gopts)
|
printer := newTerminalProgressPrinter(gopts.verbosity, term)
|
||||||
AddCleanupHandler(func(code int) (int, error) {
|
|
||||||
cleanup()
|
|
||||||
return code, nil
|
|
||||||
})
|
|
||||||
|
|
||||||
repo, err := OpenRepository(ctx, gopts)
|
cleanup := prepareCheckCache(opts, &gopts, printer)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
if !gopts.NoLock {
|
||||||
|
printer.P("create exclusive lock for repository\n")
|
||||||
|
}
|
||||||
|
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, gopts.NoLock)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
defer unlock()
|
||||||
if !gopts.NoLock {
|
|
||||||
Verbosef("create exclusive lock for repository\n")
|
|
||||||
var lock *restic.Lock
|
|
||||||
lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
|
|
||||||
defer unlockRepo(lock)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
chkr := checker.New(repo, opts.CheckUnused)
|
chkr := checker.New(repo, opts.CheckUnused)
|
||||||
err = chkr.LoadSnapshots(ctx)
|
err = chkr.LoadSnapshots(ctx)
|
||||||
|
@ -225,37 +229,48 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
Verbosef("load indexes\n")
|
printer.P("load indexes\n")
|
||||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
|
||||||
hints, errs := chkr.LoadIndex(ctx, bar)
|
hints, errs := chkr.LoadIndex(ctx, bar)
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
errorsFound := false
|
errorsFound := false
|
||||||
suggestIndexRebuild := false
|
suggestIndexRebuild := false
|
||||||
|
suggestLegacyIndexRebuild := false
|
||||||
mixedFound := false
|
mixedFound := false
|
||||||
for _, hint := range hints {
|
for _, hint := range hints {
|
||||||
switch hint.(type) {
|
switch hint.(type) {
|
||||||
case *checker.ErrDuplicatePacks, *checker.ErrOldIndexFormat:
|
case *checker.ErrDuplicatePacks:
|
||||||
Printf("%v\n", hint)
|
term.Print(hint.Error())
|
||||||
suggestIndexRebuild = true
|
suggestIndexRebuild = true
|
||||||
|
case *checker.ErrOldIndexFormat:
|
||||||
|
printer.E("error: %v\n", hint)
|
||||||
|
suggestLegacyIndexRebuild = true
|
||||||
|
errorsFound = true
|
||||||
case *checker.ErrMixedPack:
|
case *checker.ErrMixedPack:
|
||||||
Printf("%v\n", hint)
|
term.Print(hint.Error())
|
||||||
mixedFound = true
|
mixedFound = true
|
||||||
default:
|
default:
|
||||||
Warnf("error: %v\n", hint)
|
printer.E("error: %v\n", hint)
|
||||||
errorsFound = true
|
errorsFound = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if suggestIndexRebuild {
|
if suggestIndexRebuild {
|
||||||
Printf("Duplicate packs/old indexes are non-critical, you can run `restic repair index' to correct this.\n")
|
term.Print("Duplicate packs are non-critical, you can run `restic repair index' to correct this.\n")
|
||||||
|
}
|
||||||
|
if suggestLegacyIndexRebuild {
|
||||||
|
printer.E("Found indexes using the legacy format, you must run `restic repair index' to correct this.\n")
|
||||||
}
|
}
|
||||||
if mixedFound {
|
if mixedFound {
|
||||||
Printf("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
|
term.Print("Mixed packs with tree and data blobs are non-critical, you can run `restic prune` to correct this.\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(errs) > 0 {
|
if len(errs) > 0 {
|
||||||
for _, err := range errs {
|
for _, err := range errs {
|
||||||
Warnf("error: %v\n", err)
|
printer.E("error: %v\n", err)
|
||||||
}
|
}
|
||||||
return errors.Fatal("LoadIndex returned errors")
|
return errors.Fatal("LoadIndex returned errors")
|
||||||
}
|
}
|
||||||
|
@ -263,33 +278,36 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
|
||||||
orphanedPacks := 0
|
orphanedPacks := 0
|
||||||
errChan := make(chan error)
|
errChan := make(chan error)
|
||||||
|
|
||||||
Verbosef("check all packs\n")
|
printer.P("check all packs\n")
|
||||||
go chkr.Packs(ctx, errChan)
|
go chkr.Packs(ctx, errChan)
|
||||||
|
|
||||||
for err := range errChan {
|
for err := range errChan {
|
||||||
if checker.IsOrphanedPack(err) {
|
if checker.IsOrphanedPack(err) {
|
||||||
orphanedPacks++
|
orphanedPacks++
|
||||||
Verbosef("%v\n", err)
|
printer.P("%v\n", err)
|
||||||
} else if err == checker.ErrLegacyLayout {
|
} else if err == checker.ErrLegacyLayout {
|
||||||
Verbosef("repository still uses the S3 legacy layout\nPlease run `restic migrate s3legacy` to correct this.\n")
|
printer.P("repository still uses the S3 legacy layout\nPlease run `restic migrate s3legacy` to correct this.\n")
|
||||||
} else {
|
} else {
|
||||||
errorsFound = true
|
errorsFound = true
|
||||||
Warnf("%v\n", err)
|
printer.E("%v\n", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if orphanedPacks > 0 {
|
if orphanedPacks > 0 {
|
||||||
Verbosef("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks)
|
printer.P("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks)
|
||||||
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
Verbosef("check snapshots, trees and blobs\n")
|
printer.P("check snapshots, trees and blobs\n")
|
||||||
errChan = make(chan error)
|
errChan = make(chan error)
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
bar := newProgressMax(!gopts.Quiet, 0, "snapshots")
|
bar := newTerminalProgressMax(!gopts.Quiet, 0, "snapshots", term)
|
||||||
defer bar.Done()
|
defer bar.Done()
|
||||||
chkr.Structure(ctx, bar, errChan)
|
chkr.Structure(ctx, bar, errChan)
|
||||||
}()
|
}()
|
||||||
|
@ -297,16 +315,12 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
|
||||||
for err := range errChan {
|
for err := range errChan {
|
||||||
errorsFound = true
|
errorsFound = true
|
||||||
if e, ok := err.(*checker.TreeError); ok {
|
if e, ok := err.(*checker.TreeError); ok {
|
||||||
var clean string
|
printer.E("error for tree %v:\n", e.ID.Str())
|
||||||
if stdoutCanUpdateStatus() {
|
|
||||||
clean = clearLine(0)
|
|
||||||
}
|
|
||||||
Warnf(clean+"error for tree %v:\n", e.ID.Str())
|
|
||||||
for _, treeErr := range e.Errors {
|
for _, treeErr := range e.Errors {
|
||||||
Warnf(" %v\n", treeErr)
|
printer.E(" %v\n", treeErr)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
Warnf("error: %v\n", err)
|
printer.E("error: %v\n", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -314,10 +328,17 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
|
||||||
// Must happen after `errChan` is read from in the above loop to avoid
|
// Must happen after `errChan` is read from in the above loop to avoid
|
||||||
// deadlocking in the case of errors.
|
// deadlocking in the case of errors.
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
if opts.CheckUnused {
|
if opts.CheckUnused {
|
||||||
for _, id := range chkr.UnusedBlobs(ctx) {
|
unused, err := chkr.UnusedBlobs(ctx)
|
||||||
Verbosef("unused blob %v\n", id)
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for _, id := range unused {
|
||||||
|
printer.P("unused blob %v\n", id)
|
||||||
errorsFound = true
|
errorsFound = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -325,7 +346,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
|
||||||
doReadData := func(packs map[restic.ID]int64) {
|
doReadData := func(packs map[restic.ID]int64) {
|
||||||
packCount := uint64(len(packs))
|
packCount := uint64(len(packs))
|
||||||
|
|
||||||
p := newProgressMax(!gopts.Quiet, packCount, "packs")
|
p := newTerminalProgressMax(!gopts.Quiet, packCount, "packs", term)
|
||||||
errChan := make(chan error)
|
errChan := make(chan error)
|
||||||
|
|
||||||
go chkr.ReadPacks(ctx, packs, p, errChan)
|
go chkr.ReadPacks(ctx, packs, p, errChan)
|
||||||
|
@ -334,29 +355,27 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
|
||||||
|
|
||||||
for err := range errChan {
|
for err := range errChan {
|
||||||
errorsFound = true
|
errorsFound = true
|
||||||
Warnf("%v\n", err)
|
printer.E("%v\n", err)
|
||||||
if err, ok := err.(*checker.ErrPackData); ok {
|
if err, ok := err.(*repository.ErrPackData); ok {
|
||||||
if strings.Contains(err.Error(), "wrong data returned, hash is") {
|
salvagePacks = append(salvagePacks, err.PackID)
|
||||||
salvagePacks = append(salvagePacks, err.PackID)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
p.Done()
|
p.Done()
|
||||||
|
|
||||||
if len(salvagePacks) > 0 {
|
if len(salvagePacks) > 0 {
|
||||||
Warnf("\nThe repository contains pack files with damaged blobs. These blobs must be removed to repair the repository. This can be done using the following commands:\n\n")
|
printer.E("\nThe repository contains pack files with damaged blobs. These blobs must be removed to repair the repository. This can be done using the following commands. Please read the troubleshooting guide at https://restic.readthedocs.io/en/stable/077_troubleshooting.html first.\n\n")
|
||||||
var strIds []string
|
var strIDs []string
|
||||||
for _, id := range salvagePacks {
|
for _, id := range salvagePacks {
|
||||||
strIds = append(strIds, id.String())
|
strIDs = append(strIDs, id.String())
|
||||||
}
|
}
|
||||||
Warnf("RESTIC_FEATURES=repair-packs-v1 restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIds, " "))
|
printer.E("restic repair packs %v\nrestic repair snapshots --forget\n\n", strings.Join(strIDs, " "))
|
||||||
Warnf("Corrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n")
|
printer.E("Corrupted blobs are either caused by hardware problems or bugs in restic. Please open an issue at https://github.com/restic/restic/issues/new/choose for further troubleshooting!\n")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case opts.ReadData:
|
case opts.ReadData:
|
||||||
Verbosef("read all data\n")
|
printer.P("read all data\n")
|
||||||
doReadData(selectPacksByBucket(chkr.GetPacks(), 1, 1))
|
doReadData(selectPacksByBucket(chkr.GetPacks(), 1, 1))
|
||||||
case opts.ReadDataSubset != "":
|
case opts.ReadDataSubset != "":
|
||||||
var packs map[restic.ID]int64
|
var packs map[restic.ID]int64
|
||||||
|
@ -366,12 +385,12 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
|
||||||
totalBuckets := dataSubset[1]
|
totalBuckets := dataSubset[1]
|
||||||
packs = selectPacksByBucket(chkr.GetPacks(), bucket, totalBuckets)
|
packs = selectPacksByBucket(chkr.GetPacks(), bucket, totalBuckets)
|
||||||
packCount := uint64(len(packs))
|
packCount := uint64(len(packs))
|
||||||
Verbosef("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets)
|
printer.P("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets)
|
||||||
} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
|
} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
|
||||||
percentage, err := parsePercentage(opts.ReadDataSubset)
|
percentage, err := parsePercentage(opts.ReadDataSubset)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage)
|
packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage)
|
||||||
Verbosef("read %.1f%% of data packs\n", percentage)
|
printer.P("read %.1f%% of data packs\n", percentage)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
repoSize := int64(0)
|
repoSize := int64(0)
|
||||||
|
@ -387,7 +406,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
|
||||||
subsetSize = repoSize
|
subsetSize = repoSize
|
||||||
}
|
}
|
||||||
packs = selectRandomPacksByFileSize(chkr.GetPacks(), subsetSize, repoSize)
|
packs = selectRandomPacksByFileSize(chkr.GetPacks(), subsetSize, repoSize)
|
||||||
Verbosef("read %d bytes of data packs\n", subsetSize)
|
printer.P("read %d bytes of data packs\n", subsetSize)
|
||||||
}
|
}
|
||||||
if packs == nil {
|
if packs == nil {
|
||||||
return errors.Fatal("internal error: failed to select packs to check")
|
return errors.Fatal("internal error: failed to select packs to check")
|
||||||
|
@ -395,11 +414,14 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
|
||||||
doReadData(packs)
|
doReadData(packs)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
if errorsFound {
|
if errorsFound {
|
||||||
return errors.Fatal("repository contains errors")
|
return errors.Fatal("repository contains errors")
|
||||||
}
|
}
|
||||||
|
printer.P("no errors were found\n")
|
||||||
Verbosef("no errors were found\n")
|
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
@ -417,7 +439,7 @@ func selectPacksByBucket(allPacks map[restic.ID]int64, bucket, totalBuckets uint
|
||||||
return packs
|
return packs
|
||||||
}
|
}
|
||||||
|
|
||||||
// selectRandomPacksByPercentage selects the given percentage of packs which are randomly choosen.
|
// selectRandomPacksByPercentage selects the given percentage of packs which are randomly chosen.
|
||||||
func selectRandomPacksByPercentage(allPacks map[restic.ID]int64, percentage float64) map[restic.ID]int64 {
|
func selectRandomPacksByPercentage(allPacks map[restic.ID]int64, percentage float64) map[restic.ID]int64 {
|
||||||
packCount := len(allPacks)
|
packCount := len(allPacks)
|
||||||
packsToCheck := int(float64(packCount) * (percentage / 100.0))
|
packsToCheck := int(float64(packCount) * (percentage / 100.0))
|
||||||
|
|
|
@ -1,10 +1,12 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"context"
|
"context"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
rtest "github.com/restic/restic/internal/test"
|
rtest "github.com/restic/restic/internal/test"
|
||||||
|
"github.com/restic/restic/internal/ui/termstatus"
|
||||||
)
|
)
|
||||||
|
|
||||||
func testRunCheck(t testing.TB, gopts GlobalOptions) {
|
func testRunCheck(t testing.TB, gopts GlobalOptions) {
|
||||||
|
@ -23,12 +25,14 @@ func testRunCheckMustFail(t testing.TB, gopts GlobalOptions) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func testRunCheckOutput(gopts GlobalOptions, checkUnused bool) (string, error) {
|
func testRunCheckOutput(gopts GlobalOptions, checkUnused bool) (string, error) {
|
||||||
buf, err := withCaptureStdout(func() error {
|
buf := bytes.NewBuffer(nil)
|
||||||
|
gopts.stdout = buf
|
||||||
|
err := withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
|
||||||
opts := CheckOptions{
|
opts := CheckOptions{
|
||||||
ReadData: true,
|
ReadData: true,
|
||||||
CheckUnused: checkUnused,
|
CheckUnused: checkUnused,
|
||||||
}
|
}
|
||||||
return runCheck(context.TODO(), opts, gopts, nil)
|
return runCheck(context.TODO(), opts, gopts, nil, term)
|
||||||
})
|
})
|
||||||
return buf.String(), err
|
return buf.String(), err
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,12 +1,16 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"io/fs"
|
||||||
"math"
|
"math"
|
||||||
|
"os"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
rtest "github.com/restic/restic/internal/test"
|
rtest "github.com/restic/restic/internal/test"
|
||||||
|
"github.com/restic/restic/internal/ui/progress"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestParsePercentage(t *testing.T) {
|
func TestParsePercentage(t *testing.T) {
|
||||||
|
@ -71,7 +75,7 @@ func TestSelectPacksByBucket(t *testing.T) {
|
||||||
var testPacks = make(map[restic.ID]int64)
|
var testPacks = make(map[restic.ID]int64)
|
||||||
for i := 1; i <= 10; i++ {
|
for i := 1; i <= 10; i++ {
|
||||||
id := restic.NewRandomID()
|
id := restic.NewRandomID()
|
||||||
// ensure relevant part of generated id is reproducable
|
// ensure relevant part of generated id is reproducible
|
||||||
id[0] = byte(i)
|
id[0] = byte(i)
|
||||||
testPacks[id] = 0
|
testPacks[id] = 0
|
||||||
}
|
}
|
||||||
|
@ -124,7 +128,7 @@ func TestSelectRandomPacksByPercentage(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSelectNoRandomPacksByPercentage(t *testing.T) {
|
func TestSelectNoRandomPacksByPercentage(t *testing.T) {
|
||||||
// that the a repository without pack files works
|
// that the repository without pack files works
|
||||||
var testPacks = make(map[restic.ID]int64)
|
var testPacks = make(map[restic.ID]int64)
|
||||||
selectedPacks := selectRandomPacksByPercentage(testPacks, 10.0)
|
selectedPacks := selectRandomPacksByPercentage(testPacks, 10.0)
|
||||||
rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
|
rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
|
||||||
|
@ -158,8 +162,70 @@ func TestSelectRandomPacksByFileSize(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestSelectNoRandomPacksByFileSize(t *testing.T) {
|
func TestSelectNoRandomPacksByFileSize(t *testing.T) {
|
||||||
// that the a repository without pack files works
|
// that the repository without pack files works
|
||||||
var testPacks = make(map[restic.ID]int64)
|
var testPacks = make(map[restic.ID]int64)
|
||||||
selectedPacks := selectRandomPacksByFileSize(testPacks, 10, 500)
|
selectedPacks := selectRandomPacksByFileSize(testPacks, 10, 500)
|
||||||
rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
|
rtest.Assert(t, len(selectedPacks) == 0, "Expected 0 selected packs")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func checkIfFileWithSimilarNameExists(files []fs.DirEntry, fileName string) bool {
|
||||||
|
found := false
|
||||||
|
for _, file := range files {
|
||||||
|
if file.IsDir() {
|
||||||
|
dirName := file.Name()
|
||||||
|
if strings.Contains(dirName, fileName) {
|
||||||
|
found = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return found
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPrepareCheckCache(t *testing.T) {
|
||||||
|
// Create a temporary directory for the cache
|
||||||
|
tmpDirBase := t.TempDir()
|
||||||
|
|
||||||
|
testCases := []struct {
|
||||||
|
opts CheckOptions
|
||||||
|
withValidCache bool
|
||||||
|
}{
|
||||||
|
{CheckOptions{WithCache: true}, true}, // Shouldn't create temp directory
|
||||||
|
{CheckOptions{WithCache: false}, true}, // Should create temp directory
|
||||||
|
{CheckOptions{WithCache: false}, false}, // Should create cache directory first, then temp directory
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, testCase := range testCases {
|
||||||
|
t.Run("", func(t *testing.T) {
|
||||||
|
if !testCase.withValidCache {
|
||||||
|
// remove tmpDirBase to simulate non-existing cache directory
|
||||||
|
err := os.Remove(tmpDirBase)
|
||||||
|
rtest.OK(t, err)
|
||||||
|
}
|
||||||
|
gopts := GlobalOptions{CacheDir: tmpDirBase}
|
||||||
|
cleanup := prepareCheckCache(testCase.opts, &gopts, &progress.NoopPrinter{})
|
||||||
|
files, err := os.ReadDir(tmpDirBase)
|
||||||
|
rtest.OK(t, err)
|
||||||
|
|
||||||
|
if !testCase.opts.WithCache {
|
||||||
|
// If using a temporary cache directory, the cache directory should exist
|
||||||
|
// listing all directories inside tmpDirBase (cacheDir)
|
||||||
|
// one directory should be tmpDir created by prepareCheckCache with 'restic-check-cache-' in path
|
||||||
|
found := checkIfFileWithSimilarNameExists(files, "restic-check-cache-")
|
||||||
|
if !found {
|
||||||
|
t.Errorf("Expected temporary directory to exist, but it does not")
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// If not using the cache, the temp directory should not exist
|
||||||
|
rtest.Assert(t, len(files) == 0, "expected cache directory not to exist, but it does: %v", files)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Call the cleanup function to remove the temporary cache directory
|
||||||
|
cleanup()
|
||||||
|
|
||||||
|
// Verify that the cache directory has been removed
|
||||||
|
files, err = os.ReadDir(tmpDirBase)
|
||||||
|
rtest.OK(t, err)
|
||||||
|
rtest.Assert(t, len(files) == 0, "Expected cache directory to be removed, but it still exists: %v", files)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
@ -4,7 +4,6 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/backend"
|
|
||||||
"github.com/restic/restic/internal/debug"
|
"github.com/restic/restic/internal/debug"
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
"github.com/restic/restic/internal/repository"
|
"github.com/restic/restic/internal/repository"
|
||||||
|
@ -54,7 +53,7 @@ func init() {
|
||||||
}
|
}
|
||||||
|
|
||||||
func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error {
|
func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error {
|
||||||
secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination")
|
secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(ctx, opts.secondaryRepoOptions, gopts, "destination")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -63,37 +62,24 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
|
||||||
gopts, secondaryGopts = secondaryGopts, gopts
|
gopts, secondaryGopts = secondaryGopts, gopts
|
||||||
}
|
}
|
||||||
|
|
||||||
srcRepo, err := OpenRepository(ctx, gopts)
|
ctx, srcRepo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
|
|
||||||
|
ctx, dstRepo, unlock, err := openWithAppendLock(ctx, secondaryGopts, false)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
|
|
||||||
|
srcSnapshotLister, err := restic.MemorizeList(ctx, srcRepo, restic.SnapshotFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
dstRepo, err := OpenRepository(ctx, secondaryGopts)
|
dstSnapshotLister, err := restic.MemorizeList(ctx, dstRepo, restic.SnapshotFile)
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if !gopts.NoLock {
|
|
||||||
var srcLock *restic.Lock
|
|
||||||
srcLock, ctx, err = lockRepo(ctx, srcRepo, gopts.RetryLock, gopts.JSON)
|
|
||||||
defer unlockRepo(srcLock)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
dstLock, ctx, err := lockRepo(ctx, dstRepo, gopts.RetryLock, gopts.JSON)
|
|
||||||
defer unlockRepo(dstLock)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
srcSnapshotLister, err := backend.MemorizeList(ctx, srcRepo.Backend(), restic.SnapshotFile)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
dstSnapshotLister, err := backend.MemorizeList(ctx, dstRepo.Backend(), restic.SnapshotFile)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -117,6 +103,9 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
|
||||||
// also consider identical snapshot copies
|
// also consider identical snapshot copies
|
||||||
dstSnapshotByOriginal[*sn.ID()] = append(dstSnapshotByOriginal[*sn.ID()], sn)
|
dstSnapshotByOriginal[*sn.ID()] = append(dstSnapshotByOriginal[*sn.ID()], sn)
|
||||||
}
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
// remember already processed trees across all snapshots
|
// remember already processed trees across all snapshots
|
||||||
visitedTrees := restic.NewIDSet()
|
visitedTrees := restic.NewIDSet()
|
||||||
|
@ -127,11 +116,12 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
|
||||||
if sn.Original != nil {
|
if sn.Original != nil {
|
||||||
srcOriginal = *sn.Original
|
srcOriginal = *sn.Original
|
||||||
}
|
}
|
||||||
|
|
||||||
if originalSns, ok := dstSnapshotByOriginal[srcOriginal]; ok {
|
if originalSns, ok := dstSnapshotByOriginal[srcOriginal]; ok {
|
||||||
isCopy := false
|
isCopy := false
|
||||||
for _, originalSn := range originalSns {
|
for _, originalSn := range originalSns {
|
||||||
if similarSnapshots(originalSn, sn) {
|
if similarSnapshots(originalSn, sn) {
|
||||||
Verboseff("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
|
Verboseff("\n%v\n", sn)
|
||||||
Verboseff("skipping source snapshot %s, was already copied to snapshot %s\n", sn.ID().Str(), originalSn.ID().Str())
|
Verboseff("skipping source snapshot %s, was already copied to snapshot %s\n", sn.ID().Str(), originalSn.ID().Str())
|
||||||
isCopy = true
|
isCopy = true
|
||||||
break
|
break
|
||||||
|
@ -141,7 +131,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Verbosef("\nsnapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
|
Verbosef("\n%v\n", sn)
|
||||||
Verbosef(" copy started, this may take a while...\n")
|
Verbosef(" copy started, this may take a while...\n")
|
||||||
if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, gopts.Quiet); err != nil {
|
if err := copyTree(ctx, srcRepo, dstRepo, visitedTrees, *sn.Tree, gopts.Quiet); err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -160,7 +150,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
|
||||||
}
|
}
|
||||||
Verbosef("snapshot %s saved\n", newID.Str())
|
Verbosef("snapshot %s saved\n", newID.Str())
|
||||||
}
|
}
|
||||||
return nil
|
return ctx.Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool {
|
func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool {
|
||||||
|
@ -197,7 +187,7 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
|
||||||
packList := restic.NewIDSet()
|
packList := restic.NewIDSet()
|
||||||
|
|
||||||
enqueue := func(h restic.BlobHandle) {
|
enqueue := func(h restic.BlobHandle) {
|
||||||
pb := srcRepo.Index().Lookup(h)
|
pb := srcRepo.LookupBlob(h.Type, h.ID)
|
||||||
copyBlobs.Insert(h)
|
copyBlobs.Insert(h)
|
||||||
for _, p := range pb {
|
for _, p := range pb {
|
||||||
packList.Insert(p.PackID)
|
packList.Insert(p.PackID)
|
||||||
|
@ -212,7 +202,7 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
|
||||||
|
|
||||||
// Do we already have this tree blob?
|
// Do we already have this tree blob?
|
||||||
treeHandle := restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}
|
treeHandle := restic.BlobHandle{ID: tree.ID, Type: restic.TreeBlob}
|
||||||
if !dstRepo.Index().Has(treeHandle) {
|
if _, ok := dstRepo.LookupBlobSize(treeHandle.Type, treeHandle.ID); !ok {
|
||||||
// copy raw tree bytes to avoid problems if the serialization changes
|
// copy raw tree bytes to avoid problems if the serialization changes
|
||||||
enqueue(treeHandle)
|
enqueue(treeHandle)
|
||||||
}
|
}
|
||||||
|
@ -222,7 +212,7 @@ func copyTree(ctx context.Context, srcRepo restic.Repository, dstRepo restic.Rep
|
||||||
// Copy the blobs for this file.
|
// Copy the blobs for this file.
|
||||||
for _, blobID := range entry.Content {
|
for _, blobID := range entry.Content {
|
||||||
h := restic.BlobHandle{Type: restic.DataBlob, ID: blobID}
|
h := restic.BlobHandle{Type: restic.DataBlob, ID: blobID}
|
||||||
if !dstRepo.Index().Has(h) {
|
if _, ok := dstRepo.LookupBlobSize(h.Type, h.ID); !ok {
|
||||||
enqueue(h)
|
enqueue(h)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -13,10 +13,12 @@ func testRunCopy(t testing.TB, srcGopts GlobalOptions, dstGopts GlobalOptions) {
|
||||||
gopts := srcGopts
|
gopts := srcGopts
|
||||||
gopts.Repo = dstGopts.Repo
|
gopts.Repo = dstGopts.Repo
|
||||||
gopts.password = dstGopts.password
|
gopts.password = dstGopts.password
|
||||||
|
gopts.InsecureNoPassword = dstGopts.InsecureNoPassword
|
||||||
copyOpts := CopyOptions{
|
copyOpts := CopyOptions{
|
||||||
secondaryRepoOptions: secondaryRepoOptions{
|
secondaryRepoOptions: secondaryRepoOptions{
|
||||||
Repo: srcGopts.Repo,
|
Repo: srcGopts.Repo,
|
||||||
password: srcGopts.password,
|
password: srcGopts.password,
|
||||||
|
InsecureNoPassword: srcGopts.InsecureNoPassword,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -134,3 +136,22 @@ func TestCopyUnstableJSON(t *testing.T) {
|
||||||
testRunCheck(t, env2.gopts)
|
testRunCheck(t, env2.gopts)
|
||||||
testListSnapshots(t, env2.gopts, 1)
|
testListSnapshots(t, env2.gopts, 1)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCopyToEmptyPassword(t *testing.T) {
|
||||||
|
env, cleanup := withTestEnvironment(t)
|
||||||
|
defer cleanup()
|
||||||
|
env2, cleanup2 := withTestEnvironment(t)
|
||||||
|
defer cleanup2()
|
||||||
|
env2.gopts.password = ""
|
||||||
|
env2.gopts.InsecureNoPassword = true
|
||||||
|
|
||||||
|
testSetupBackupData(t, env)
|
||||||
|
testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, BackupOptions{}, env.gopts)
|
||||||
|
|
||||||
|
testRunInit(t, env2.gopts)
|
||||||
|
testRunCopy(t, env.gopts, env2.gopts)
|
||||||
|
|
||||||
|
testListSnapshots(t, env.gopts, 1)
|
||||||
|
testListSnapshots(t, env2.gopts, 1)
|
||||||
|
testRunCheck(t, env2.gopts)
|
||||||
|
}
|
||||||
|
|
|
@ -20,12 +20,11 @@ import (
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
"golang.org/x/sync/errgroup"
|
"golang.org/x/sync/errgroup"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/backend"
|
|
||||||
"github.com/restic/restic/internal/crypto"
|
"github.com/restic/restic/internal/crypto"
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
"github.com/restic/restic/internal/index"
|
|
||||||
"github.com/restic/restic/internal/pack"
|
|
||||||
"github.com/restic/restic/internal/repository"
|
"github.com/restic/restic/internal/repository"
|
||||||
|
"github.com/restic/restic/internal/repository/index"
|
||||||
|
"github.com/restic/restic/internal/repository/pack"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -52,19 +51,23 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
var tryRepair bool
|
type DebugExamineOptions struct {
|
||||||
var repairByte bool
|
TryRepair bool
|
||||||
var extractPack bool
|
RepairByte bool
|
||||||
var reuploadBlobs bool
|
ExtractPack bool
|
||||||
|
ReuploadBlobs bool
|
||||||
|
}
|
||||||
|
|
||||||
|
var debugExamineOpts DebugExamineOptions
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
cmdRoot.AddCommand(cmdDebug)
|
cmdRoot.AddCommand(cmdDebug)
|
||||||
cmdDebug.AddCommand(cmdDebugDump)
|
cmdDebug.AddCommand(cmdDebugDump)
|
||||||
cmdDebug.AddCommand(cmdDebugExamine)
|
cmdDebug.AddCommand(cmdDebugExamine)
|
||||||
cmdDebugExamine.Flags().BoolVar(&extractPack, "extract-pack", false, "write blobs to the current directory")
|
cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.ExtractPack, "extract-pack", false, "write blobs to the current directory")
|
||||||
cmdDebugExamine.Flags().BoolVar(&reuploadBlobs, "reupload-blobs", false, "reupload blobs to the repository")
|
cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.ReuploadBlobs, "reupload-blobs", false, "reupload blobs to the repository")
|
||||||
cmdDebugExamine.Flags().BoolVar(&tryRepair, "try-repair", false, "try to repair broken blobs with single bit flips")
|
cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.TryRepair, "try-repair", false, "try to repair broken blobs with single bit flips")
|
||||||
cmdDebugExamine.Flags().BoolVar(&repairByte, "repair-byte", false, "try to repair broken blobs by trying bytes")
|
cmdDebugExamine.Flags().BoolVar(&debugExamineOpts.RepairByte, "repair-byte", false, "try to repair broken blobs by trying bytes")
|
||||||
}
|
}
|
||||||
|
|
||||||
func prettyPrintJSON(wr io.Writer, item interface{}) error {
|
func prettyPrintJSON(wr io.Writer, item interface{}) error {
|
||||||
|
@ -78,7 +81,7 @@ func prettyPrintJSON(wr io.Writer, item interface{}) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
|
func debugPrintSnapshots(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
|
||||||
return restic.ForAllSnapshots(ctx, repo.Backend(), repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
|
return restic.ForAllSnapshots(ctx, repo, repo, nil, func(id restic.ID, snapshot *restic.Snapshot, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -107,7 +110,7 @@ type Blob struct {
|
||||||
func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
|
func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer) error {
|
||||||
|
|
||||||
var m sync.Mutex
|
var m sync.Mutex
|
||||||
return restic.ParallelList(ctx, repo.Backend(), restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
|
return restic.ParallelList(ctx, repo, restic.PackFile, repo.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
|
||||||
blobs, _, err := repo.ListPack(ctx, id, size)
|
blobs, _, err := repo.ListPack(ctx, id, size)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Warnf("error for pack %v: %v\n", id.Str(), err)
|
Warnf("error for pack %v: %v\n", id.Str(), err)
|
||||||
|
@ -133,8 +136,8 @@ func printPacks(ctx context.Context, repo *repository.Repository, wr io.Writer)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func dumpIndexes(ctx context.Context, repo restic.Repository, wr io.Writer) error {
|
func dumpIndexes(ctx context.Context, repo restic.ListerLoaderUnpacked, wr io.Writer) error {
|
||||||
return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
|
return index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
|
||||||
Printf("index_id: %v\n", id)
|
Printf("index_id: %v\n", id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -149,19 +152,11 @@ func runDebugDump(ctx context.Context, gopts GlobalOptions, args []string) error
|
||||||
return errors.Fatal("type not specified")
|
return errors.Fatal("type not specified")
|
||||||
}
|
}
|
||||||
|
|
||||||
repo, err := OpenRepository(ctx, gopts)
|
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
defer unlock()
|
||||||
if !gopts.NoLock {
|
|
||||||
var lock *restic.Lock
|
|
||||||
lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
|
|
||||||
defer unlockRepo(lock)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
tpe := args[0]
|
tpe := args[0]
|
||||||
|
|
||||||
|
@ -196,7 +191,7 @@ var cmdDebugExamine = &cobra.Command{
|
||||||
Short: "Examine a pack file",
|
Short: "Examine a pack file",
|
||||||
DisableAutoGenTag: true,
|
DisableAutoGenTag: true,
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
return runDebugExamine(cmd.Context(), globalOptions, args)
|
return runDebugExamine(cmd.Context(), globalOptions, debugExamineOpts, args)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -290,7 +285,7 @@ func tryRepairWithBitflip(ctx context.Context, key *crypto.Key, input []byte, by
|
||||||
})
|
})
|
||||||
err := wg.Wait()
|
err := wg.Wait()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic("all go rountines can only return nil")
|
panic("all go routines can only return nil")
|
||||||
}
|
}
|
||||||
|
|
||||||
if !found {
|
if !found {
|
||||||
|
@ -315,39 +310,32 @@ func decryptUnsigned(ctx context.Context, k *crypto.Key, buf []byte) []byte {
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, list []restic.Blob) error {
|
func loadBlobs(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, packID restic.ID, list []restic.Blob) error {
|
||||||
dec, err := zstd.NewReader(nil)
|
dec, err := zstd.NewReader(nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
be := repo.Backend()
|
|
||||||
h := restic.Handle{
|
pack, err := repo.LoadRaw(ctx, restic.PackFile, packID)
|
||||||
Name: packID.String(),
|
// allow processing broken pack files
|
||||||
Type: restic.PackFile,
|
if pack == nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
wg, ctx := errgroup.WithContext(ctx)
|
wg, ctx := errgroup.WithContext(ctx)
|
||||||
|
|
||||||
if reuploadBlobs {
|
if opts.ReuploadBlobs {
|
||||||
repo.StartPackUploader(ctx, wg)
|
repo.StartPackUploader(ctx, wg)
|
||||||
}
|
}
|
||||||
|
|
||||||
wg.Go(func() error {
|
wg.Go(func() error {
|
||||||
for _, blob := range list {
|
for _, blob := range list {
|
||||||
Printf(" loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length)
|
Printf(" loading blob %v at %v (length %v)\n", blob.ID, blob.Offset, blob.Length)
|
||||||
buf := make([]byte, blob.Length)
|
if int(blob.Offset+blob.Length) > len(pack) {
|
||||||
err := be.Load(ctx, h, int(blob.Length), int64(blob.Offset), func(rd io.Reader) error {
|
Warnf("skipping truncated blob\n")
|
||||||
n, err := io.ReadFull(rd, buf)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("read error after %d bytes: %v", n, err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
Warnf("error read: %v\n", err)
|
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
buf := pack[blob.Offset : blob.Offset+blob.Length]
|
||||||
key := repo.Key()
|
key := repo.Key()
|
||||||
|
|
||||||
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
|
nonce, plaintext := buf[:key.NonceSize()], buf[key.NonceSize():]
|
||||||
|
@ -356,8 +344,8 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li
|
||||||
filePrefix := ""
|
filePrefix := ""
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Warnf("error decrypting blob: %v\n", err)
|
Warnf("error decrypting blob: %v\n", err)
|
||||||
if tryRepair || repairByte {
|
if opts.TryRepair || opts.RepairByte {
|
||||||
plaintext = tryRepairWithBitflip(ctx, key, buf, repairByte)
|
plaintext = tryRepairWithBitflip(ctx, key, buf, opts.RepairByte)
|
||||||
}
|
}
|
||||||
if plaintext != nil {
|
if plaintext != nil {
|
||||||
outputPrefix = "repaired "
|
outputPrefix = "repaired "
|
||||||
|
@ -391,13 +379,13 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li
|
||||||
Printf(" successfully %vdecrypted blob (length %v), hash is %v, ID matches\n", outputPrefix, len(plaintext), id)
|
Printf(" successfully %vdecrypted blob (length %v), hash is %v, ID matches\n", outputPrefix, len(plaintext), id)
|
||||||
prefix = "correct-"
|
prefix = "correct-"
|
||||||
}
|
}
|
||||||
if extractPack {
|
if opts.ExtractPack {
|
||||||
err = storePlainBlob(id, filePrefix+prefix, plaintext)
|
err = storePlainBlob(id, filePrefix+prefix, plaintext)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if reuploadBlobs {
|
if opts.ReuploadBlobs {
|
||||||
_, _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true)
|
_, _, _, err := repo.SaveBlob(ctx, blob.Type, plaintext, id, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -406,7 +394,7 @@ func loadBlobs(ctx context.Context, repo restic.Repository, packID restic.ID, li
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if reuploadBlobs {
|
if opts.ReuploadBlobs {
|
||||||
return repo.Flush(ctx)
|
return repo.Flush(ctx)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
@ -437,17 +425,22 @@ func storePlainBlob(id restic.ID, prefix string, plain []byte) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) error {
|
func runDebugExamine(ctx context.Context, gopts GlobalOptions, opts DebugExamineOptions, args []string) error {
|
||||||
repo, err := OpenRepository(ctx, gopts)
|
if opts.ExtractPack && gopts.NoLock {
|
||||||
|
return fmt.Errorf("--extract-pack and --no-lock are mutually exclusive")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, gopts.NoLock)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
defer unlock()
|
||||||
|
|
||||||
ids := make([]restic.ID, 0)
|
ids := make([]restic.ID, 0)
|
||||||
for _, name := range args {
|
for _, name := range args {
|
||||||
id, err := restic.ParseID(name)
|
id, err := restic.ParseID(name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
id, err = restic.Find(ctx, repo.Backend(), restic.PackFile, name)
|
id, err = restic.Find(ctx, repo, restic.PackFile, name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Warnf("error: %v\n", err)
|
Warnf("error: %v\n", err)
|
||||||
continue
|
continue
|
||||||
|
@ -460,15 +453,6 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er
|
||||||
return errors.Fatal("no pack files to examine")
|
return errors.Fatal("no pack files to examine")
|
||||||
}
|
}
|
||||||
|
|
||||||
if !gopts.NoLock {
|
|
||||||
var lock *restic.Lock
|
|
||||||
lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
|
|
||||||
defer unlockRepo(lock)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||||
err = repo.LoadIndex(ctx, bar)
|
err = repo.LoadIndex(ctx, bar)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -476,7 +460,7 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, id := range ids {
|
for _, id := range ids {
|
||||||
err := examinePack(ctx, repo, id)
|
err := examinePack(ctx, opts, repo, id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Warnf("error: %v\n", err)
|
Warnf("error: %v\n", err)
|
||||||
}
|
}
|
||||||
|
@ -487,23 +471,15 @@ func runDebugExamine(ctx context.Context, gopts GlobalOptions, args []string) er
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) error {
|
func examinePack(ctx context.Context, opts DebugExamineOptions, repo restic.Repository, id restic.ID) error {
|
||||||
Printf("examine %v\n", id)
|
Printf("examine %v\n", id)
|
||||||
|
|
||||||
h := restic.Handle{
|
buf, err := repo.LoadRaw(ctx, restic.PackFile, id)
|
||||||
Type: restic.PackFile,
|
// also process damaged pack files
|
||||||
Name: id.String(),
|
if buf == nil {
|
||||||
}
|
|
||||||
fi, err := repo.Backend().Stat(ctx, h)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
Printf(" file size is %v\n", fi.Size)
|
|
||||||
|
|
||||||
buf, err := backend.LoadAll(ctx, nil, repo.Backend(), h)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
Printf(" file size is %v\n", len(buf))
|
||||||
gotID := restic.Hash(buf)
|
gotID := restic.Hash(buf)
|
||||||
if !id.Equal(gotID) {
|
if !id.Equal(gotID) {
|
||||||
Printf(" wanted hash %v, got %v\n", id, gotID)
|
Printf(" wanted hash %v, got %v\n", id, gotID)
|
||||||
|
@ -516,15 +492,15 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro
|
||||||
|
|
||||||
blobsLoaded := false
|
blobsLoaded := false
|
||||||
// examine all data the indexes have for the pack file
|
// examine all data the indexes have for the pack file
|
||||||
for b := range repo.Index().ListPacks(ctx, restic.NewIDSet(id)) {
|
for b := range repo.ListPacksFromIndex(ctx, restic.NewIDSet(id)) {
|
||||||
blobs := b.Blobs
|
blobs := b.Blobs
|
||||||
if len(blobs) == 0 {
|
if len(blobs) == 0 {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
checkPackSize(blobs, fi.Size)
|
checkPackSize(blobs, len(buf))
|
||||||
|
|
||||||
err = loadBlobs(ctx, repo, id, blobs)
|
err = loadBlobs(ctx, opts, repo, id, blobs)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Warnf("error: %v\n", err)
|
Warnf("error: %v\n", err)
|
||||||
} else {
|
} else {
|
||||||
|
@ -535,19 +511,19 @@ func examinePack(ctx context.Context, repo restic.Repository, id restic.ID) erro
|
||||||
Printf(" ========================================\n")
|
Printf(" ========================================\n")
|
||||||
Printf(" inspect the pack itself\n")
|
Printf(" inspect the pack itself\n")
|
||||||
|
|
||||||
blobs, _, err := repo.ListPack(ctx, id, fi.Size)
|
blobs, _, err := repo.ListPack(ctx, id, int64(len(buf)))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("pack %v: %v", id.Str(), err)
|
return fmt.Errorf("pack %v: %v", id.Str(), err)
|
||||||
}
|
}
|
||||||
checkPackSize(blobs, fi.Size)
|
checkPackSize(blobs, len(buf))
|
||||||
|
|
||||||
if !blobsLoaded {
|
if !blobsLoaded {
|
||||||
return loadBlobs(ctx, repo, id, blobs)
|
return loadBlobs(ctx, opts, repo, id, blobs)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkPackSize(blobs []restic.Blob, fileSize int64) {
|
func checkPackSize(blobs []restic.Blob, fileSize int) {
|
||||||
// track current size and offset
|
// track current size and offset
|
||||||
var size, offset uint64
|
var size, offset uint64
|
||||||
|
|
||||||
|
|
|
@ -7,7 +7,6 @@ import (
|
||||||
"reflect"
|
"reflect"
|
||||||
"sort"
|
"sort"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/backend"
|
|
||||||
"github.com/restic/restic/internal/debug"
|
"github.com/restic/restic/internal/debug"
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
|
@ -28,6 +27,10 @@ directory:
|
||||||
* U The metadata (access mode, timestamps, ...) for the item was updated
|
* U The metadata (access mode, timestamps, ...) for the item was updated
|
||||||
* M The file's content was modified
|
* M The file's content was modified
|
||||||
* T The type was changed, e.g. a file was made a symlink
|
* T The type was changed, e.g. a file was made a symlink
|
||||||
|
* ? Bitrot detected: The file's content has changed but all metadata is the same
|
||||||
|
|
||||||
|
Metadata comparison will likely not work if a backup was created using the
|
||||||
|
'--ignore-inode' or '--ignore-ctime' option.
|
||||||
|
|
||||||
To only compare files in specific subfolders, you can use the
|
To only compare files in specific subfolders, you can use the
|
||||||
"<snapshotID>:<subfolder>" syntax, where "subfolder" is a path within the
|
"<snapshotID>:<subfolder>" syntax, where "subfolder" is a path within the
|
||||||
|
@ -58,7 +61,7 @@ func init() {
|
||||||
f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata")
|
f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata")
|
||||||
}
|
}
|
||||||
|
|
||||||
func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository, desc string) (*restic.Snapshot, string, error) {
|
func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.LoaderUnpacked, desc string) (*restic.Snapshot, string, error) {
|
||||||
sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc)
|
sn, subfolder, err := restic.FindSnapshot(ctx, be, repo, desc)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, "", errors.Fatal(err.Error())
|
return nil, "", errors.Fatal(err.Error())
|
||||||
|
@ -68,7 +71,7 @@ func loadSnapshot(ctx context.Context, be restic.Lister, repo restic.Repository,
|
||||||
|
|
||||||
// Comparer collects all things needed to compare two snapshots.
|
// Comparer collects all things needed to compare two snapshots.
|
||||||
type Comparer struct {
|
type Comparer struct {
|
||||||
repo restic.Repository
|
repo restic.BlobLoader
|
||||||
opts DiffOptions
|
opts DiffOptions
|
||||||
printChange func(change *Change)
|
printChange func(change *Change)
|
||||||
}
|
}
|
||||||
|
@ -144,7 +147,7 @@ type DiffStatsContainer struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
// updateBlobs updates the blob counters in the stats struct.
|
// updateBlobs updates the blob counters in the stats struct.
|
||||||
func updateBlobs(repo restic.Repository, blobs restic.BlobSet, stats *DiffStat) {
|
func updateBlobs(repo restic.Loader, blobs restic.BlobSet, stats *DiffStat) {
|
||||||
for h := range blobs {
|
for h := range blobs {
|
||||||
switch h.Type {
|
switch h.Type {
|
||||||
case restic.DataBlob:
|
case restic.DataBlob:
|
||||||
|
@ -153,7 +156,7 @@ func updateBlobs(repo restic.Repository, blobs restic.BlobSet, stats *DiffStat)
|
||||||
stats.TreeBlobs++
|
stats.TreeBlobs++
|
||||||
}
|
}
|
||||||
|
|
||||||
size, found := repo.LookupBlobSize(h.ID, h.Type)
|
size, found := repo.LookupBlobSize(h.Type, h.ID)
|
||||||
if !found {
|
if !found {
|
||||||
Warnf("unable to find blob size for %v\n", h)
|
Warnf("unable to find blob size for %v\n", h)
|
||||||
continue
|
continue
|
||||||
|
@ -273,6 +276,16 @@ func (c *Comparer) diffTree(ctx context.Context, stats *DiffStatsContainer, pref
|
||||||
!reflect.DeepEqual(node1.Content, node2.Content) {
|
!reflect.DeepEqual(node1.Content, node2.Content) {
|
||||||
mod += "M"
|
mod += "M"
|
||||||
stats.ChangedFiles++
|
stats.ChangedFiles++
|
||||||
|
|
||||||
|
node1NilContent := *node1
|
||||||
|
node2NilContent := *node2
|
||||||
|
node1NilContent.Content = nil
|
||||||
|
node2NilContent.Content = nil
|
||||||
|
// the bitrot detection may not work if `backup --ignore-inode` or `--ignore-ctime` were used
|
||||||
|
if node1NilContent.Equals(node2NilContent) {
|
||||||
|
// probable bitrot detected
|
||||||
|
mod += "?"
|
||||||
|
}
|
||||||
} else if c.opts.ShowMetadata && !node1.Equals(*node2) {
|
} else if c.opts.ShowMetadata && !node1.Equals(*node2) {
|
||||||
mod += "U"
|
mod += "U"
|
||||||
}
|
}
|
||||||
|
@ -331,22 +344,14 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
|
||||||
return errors.Fatalf("specify two snapshot IDs")
|
return errors.Fatalf("specify two snapshot IDs")
|
||||||
}
|
}
|
||||||
|
|
||||||
repo, err := OpenRepository(ctx, gopts)
|
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
defer unlock()
|
||||||
if !gopts.NoLock {
|
|
||||||
var lock *restic.Lock
|
|
||||||
lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
|
|
||||||
defer unlockRepo(lock)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// cache snapshots listing
|
// cache snapshots listing
|
||||||
be, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
|
be, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -388,7 +393,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
|
||||||
|
|
||||||
c := &Comparer{
|
c := &Comparer{
|
||||||
repo: repo,
|
repo: repo,
|
||||||
opts: diffOptions,
|
opts: opts,
|
||||||
printChange: func(change *Change) {
|
printChange: func(change *Change) {
|
||||||
Printf("%-5s%v\n", change.Modifier, change.Path)
|
Printf("%-5s%v\n", change.Modifier, change.Path)
|
||||||
},
|
},
|
||||||
|
@ -405,7 +410,7 @@ func runDiff(ctx context.Context, opts DiffOptions, gopts GlobalOptions, args []
|
||||||
}
|
}
|
||||||
|
|
||||||
if gopts.Quiet {
|
if gopts.Quiet {
|
||||||
c.printChange = func(change *Change) {}
|
c.printChange = func(_ *Change) {}
|
||||||
}
|
}
|
||||||
|
|
||||||
stats := &DiffStatsContainer{
|
stats := &DiffStatsContainer{
|
||||||
|
|
|
@ -46,6 +46,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
|
||||||
type DumpOptions struct {
|
type DumpOptions struct {
|
||||||
restic.SnapshotFilter
|
restic.SnapshotFilter
|
||||||
Archive string
|
Archive string
|
||||||
|
Target string
|
||||||
}
|
}
|
||||||
|
|
||||||
var dumpOptions DumpOptions
|
var dumpOptions DumpOptions
|
||||||
|
@ -56,6 +57,7 @@ func init() {
|
||||||
flags := cmdDump.Flags()
|
flags := cmdDump.Flags()
|
||||||
initSingleSnapshotFilter(flags, &dumpOptions.SnapshotFilter)
|
initSingleSnapshotFilter(flags, &dumpOptions.SnapshotFilter)
|
||||||
flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"")
|
flags.StringVarP(&dumpOptions.Archive, "archive", "a", "tar", "set archive `format` as \"tar\" or \"zip\"")
|
||||||
|
flags.StringVarP(&dumpOptions.Target, "target", "t", "", "write the output to target `path`")
|
||||||
}
|
}
|
||||||
|
|
||||||
func splitPath(p string) []string {
|
func splitPath(p string) []string {
|
||||||
|
@ -67,11 +69,11 @@ func splitPath(p string) []string {
|
||||||
return append(s, f)
|
return append(s, f)
|
||||||
}
|
}
|
||||||
|
|
||||||
func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repository, prefix string, pathComponents []string, d *dump.Dumper) error {
|
func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.BlobLoader, prefix string, pathComponents []string, d *dump.Dumper, canWriteArchiveFunc func() error) error {
|
||||||
// If we print / we need to assume that there are multiple nodes at that
|
// If we print / we need to assume that there are multiple nodes at that
|
||||||
// level in the tree.
|
// level in the tree.
|
||||||
if pathComponents[0] == "" {
|
if pathComponents[0] == "" {
|
||||||
if err := checkStdoutArchive(); err != nil {
|
if err := canWriteArchiveFunc(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
return d.DumpTree(ctx, tree, "/")
|
return d.DumpTree(ctx, tree, "/")
|
||||||
|
@ -91,9 +93,9 @@ func printFromTree(ctx context.Context, tree *restic.Tree, repo restic.Repositor
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Wrapf(err, "cannot load subtree for %q", item)
|
return errors.Wrapf(err, "cannot load subtree for %q", item)
|
||||||
}
|
}
|
||||||
return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d)
|
return printFromTree(ctx, subtree, repo, item, pathComponents[1:], d, canWriteArchiveFunc)
|
||||||
case dump.IsDir(node):
|
case dump.IsDir(node):
|
||||||
if err := checkStdoutArchive(); err != nil {
|
if err := canWriteArchiveFunc(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
|
subtree, err := restic.LoadTree(ctx, repo, *node.Subtree)
|
||||||
|
@ -129,25 +131,17 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
|
||||||
|
|
||||||
splittedPath := splitPath(path.Clean(pathToPrint))
|
splittedPath := splitPath(path.Clean(pathToPrint))
|
||||||
|
|
||||||
repo, err := OpenRepository(ctx, gopts)
|
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
defer unlock()
|
||||||
if !gopts.NoLock {
|
|
||||||
var lock *restic.Lock
|
|
||||||
lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
|
|
||||||
defer unlockRepo(lock)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
sn, subfolder, err := (&restic.SnapshotFilter{
|
sn, subfolder, err := (&restic.SnapshotFilter{
|
||||||
Hosts: opts.Hosts,
|
Hosts: opts.Hosts,
|
||||||
Paths: opts.Paths,
|
Paths: opts.Paths,
|
||||||
Tags: opts.Tags,
|
Tags: opts.Tags,
|
||||||
}).FindLatest(ctx, repo.Backend(), repo, snapshotIDString)
|
}).FindLatest(ctx, repo, repo, snapshotIDString)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Fatalf("failed to find snapshot: %v", err)
|
return errors.Fatalf("failed to find snapshot: %v", err)
|
||||||
}
|
}
|
||||||
|
@ -168,8 +162,24 @@ func runDump(ctx context.Context, opts DumpOptions, gopts GlobalOptions, args []
|
||||||
return errors.Fatalf("loading tree for snapshot %q failed: %v", snapshotIDString, err)
|
return errors.Fatalf("loading tree for snapshot %q failed: %v", snapshotIDString, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
d := dump.New(opts.Archive, repo, os.Stdout)
|
outputFileWriter := os.Stdout
|
||||||
err = printFromTree(ctx, tree, repo, "/", splittedPath, d)
|
canWriteArchiveFunc := checkStdoutArchive
|
||||||
|
|
||||||
|
if opts.Target != "" {
|
||||||
|
file, err := os.Create(opts.Target)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("cannot dump to file: %w", err)
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
_ = file.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
outputFileWriter = file
|
||||||
|
canWriteArchiveFunc = func() error { return nil }
|
||||||
|
}
|
||||||
|
|
||||||
|
d := dump.New(opts.Archive, repo, outputFileWriter)
|
||||||
|
err = printFromTree(ctx, tree, repo, "/", splittedPath, d, canWriteArchiveFunc)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.Fatalf("cannot dump file: %v", err)
|
return errors.Fatalf("cannot dump file: %v", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,58 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/restic/restic/internal/errors"
|
||||||
|
"github.com/restic/restic/internal/feature"
|
||||||
|
"github.com/restic/restic/internal/ui/table"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
var featuresCmd = &cobra.Command{
|
||||||
|
Use: "features",
|
||||||
|
Short: "Print list of feature flags",
|
||||||
|
Long: `
|
||||||
|
The "features" command prints a list of supported feature flags.
|
||||||
|
|
||||||
|
To pass feature flags to restic, set the RESTIC_FEATURES environment variable
|
||||||
|
to "featureA=true,featureB=false". Specifying an unknown feature flag is an error.
|
||||||
|
|
||||||
|
A feature can either be in alpha, beta, stable or deprecated state.
|
||||||
|
An _alpha_ feature is disabled by default and may change in arbitrary ways between restic versions or be removed.
|
||||||
|
A _beta_ feature is enabled by default, but still can change in minor ways or be removed.
|
||||||
|
A _stable_ feature is always enabled and cannot be disabled. The flag will be removed in a future restic version.
|
||||||
|
A _deprecated_ feature is always disabled and cannot be enabled. The flag will be removed in a future restic version.
|
||||||
|
|
||||||
|
EXIT STATUS
|
||||||
|
===========
|
||||||
|
|
||||||
|
Exit status is 0 if the command was successful, and non-zero if there was any error.
|
||||||
|
`,
|
||||||
|
Hidden: true,
|
||||||
|
DisableAutoGenTag: true,
|
||||||
|
RunE: func(_ *cobra.Command, args []string) error {
|
||||||
|
if len(args) != 0 {
|
||||||
|
return errors.Fatal("the feature command expects no arguments")
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("All Feature Flags:\n")
|
||||||
|
flags := feature.Flag.List()
|
||||||
|
|
||||||
|
tab := table.New()
|
||||||
|
tab.AddColumn("Name", "{{ .Name }}")
|
||||||
|
tab.AddColumn("Type", "{{ .Type }}")
|
||||||
|
tab.AddColumn("Default", "{{ .Default }}")
|
||||||
|
tab.AddColumn("Description", "{{ .Description }}")
|
||||||
|
|
||||||
|
for _, flag := range flags {
|
||||||
|
tab.AddRow(flag)
|
||||||
|
}
|
||||||
|
return tab.Write(globalOptions.stdout)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
cmdRoot.AddCommand(featuresCmd)
|
||||||
|
}
|
|
@ -9,7 +9,6 @@ import (
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/backend"
|
|
||||||
"github.com/restic/restic/internal/debug"
|
"github.com/restic/restic/internal/debug"
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
"github.com/restic/restic/internal/filter"
|
"github.com/restic/restic/internal/filter"
|
||||||
|
@ -127,6 +126,7 @@ func (s *statefulOutput) PrintPatternJSON(path string, node *restic.Node) {
|
||||||
// Make the following attributes disappear
|
// Make the following attributes disappear
|
||||||
Name byte `json:"name,omitempty"`
|
Name byte `json:"name,omitempty"`
|
||||||
ExtendedAttributes byte `json:"extended_attributes,omitempty"`
|
ExtendedAttributes byte `json:"extended_attributes,omitempty"`
|
||||||
|
GenericAttributes byte `json:"generic_attributes,omitempty"`
|
||||||
Device byte `json:"device,omitempty"`
|
Device byte `json:"device,omitempty"`
|
||||||
Content byte `json:"content,omitempty"`
|
Content byte `json:"content,omitempty"`
|
||||||
Subtree byte `json:"subtree,omitempty"`
|
Subtree byte `json:"subtree,omitempty"`
|
||||||
|
@ -245,13 +245,12 @@ func (s *statefulOutput) Finish() {
|
||||||
|
|
||||||
// Finder bundles information needed to find a file or directory.
|
// Finder bundles information needed to find a file or directory.
|
||||||
type Finder struct {
|
type Finder struct {
|
||||||
repo restic.Repository
|
repo restic.Repository
|
||||||
pat findPattern
|
pat findPattern
|
||||||
out statefulOutput
|
out statefulOutput
|
||||||
ignoreTrees restic.IDSet
|
blobIDs map[string]struct{}
|
||||||
blobIDs map[string]struct{}
|
treeIDs map[string]struct{}
|
||||||
treeIDs map[string]struct{}
|
itemsFound int
|
||||||
itemsFound int
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error {
|
func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error {
|
||||||
|
@ -262,17 +261,17 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
|
||||||
}
|
}
|
||||||
|
|
||||||
f.out.newsn = sn
|
f.out.newsn = sn
|
||||||
return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
|
return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
debug.Log("Error loading tree %v: %v", parentTreeID, err)
|
debug.Log("Error loading tree %v: %v", parentTreeID, err)
|
||||||
|
|
||||||
Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
|
Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
|
||||||
|
|
||||||
return false, walker.ErrSkipNode
|
return walker.ErrSkipNode
|
||||||
}
|
}
|
||||||
|
|
||||||
if node == nil {
|
if node == nil {
|
||||||
return false, nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
normalizedNodepath := nodepath
|
normalizedNodepath := nodepath
|
||||||
|
@ -285,7 +284,7 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
|
||||||
for _, pat := range f.pat.pattern {
|
for _, pat := range f.pat.pattern {
|
||||||
found, err := filter.Match(pat, normalizedNodepath)
|
found, err := filter.Match(pat, normalizedNodepath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return err
|
||||||
}
|
}
|
||||||
if found {
|
if found {
|
||||||
foundMatch = true
|
foundMatch = true
|
||||||
|
@ -293,16 +292,13 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var errIfNoMatch error
|
||||||
ignoreIfNoMatch = true
|
|
||||||
errIfNoMatch error
|
|
||||||
)
|
|
||||||
if node.Type == "dir" {
|
if node.Type == "dir" {
|
||||||
var childMayMatch bool
|
var childMayMatch bool
|
||||||
for _, pat := range f.pat.pattern {
|
for _, pat := range f.pat.pattern {
|
||||||
mayMatch, err := filter.ChildMatch(pat, normalizedNodepath)
|
mayMatch, err := filter.ChildMatch(pat, normalizedNodepath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return err
|
||||||
}
|
}
|
||||||
if mayMatch {
|
if mayMatch {
|
||||||
childMayMatch = true
|
childMayMatch = true
|
||||||
|
@ -311,31 +307,28 @@ func (f *Finder) findInSnapshot(ctx context.Context, sn *restic.Snapshot) error
|
||||||
}
|
}
|
||||||
|
|
||||||
if !childMayMatch {
|
if !childMayMatch {
|
||||||
ignoreIfNoMatch = true
|
|
||||||
errIfNoMatch = walker.ErrSkipNode
|
errIfNoMatch = walker.ErrSkipNode
|
||||||
} else {
|
|
||||||
ignoreIfNoMatch = false
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if !foundMatch {
|
if !foundMatch {
|
||||||
return ignoreIfNoMatch, errIfNoMatch
|
return errIfNoMatch
|
||||||
}
|
}
|
||||||
|
|
||||||
if !f.pat.oldest.IsZero() && node.ModTime.Before(f.pat.oldest) {
|
if !f.pat.oldest.IsZero() && node.ModTime.Before(f.pat.oldest) {
|
||||||
debug.Log(" ModTime is older than %s\n", f.pat.oldest)
|
debug.Log(" ModTime is older than %s\n", f.pat.oldest)
|
||||||
return ignoreIfNoMatch, errIfNoMatch
|
return errIfNoMatch
|
||||||
}
|
}
|
||||||
|
|
||||||
if !f.pat.newest.IsZero() && node.ModTime.After(f.pat.newest) {
|
if !f.pat.newest.IsZero() && node.ModTime.After(f.pat.newest) {
|
||||||
debug.Log(" ModTime is newer than %s\n", f.pat.newest)
|
debug.Log(" ModTime is newer than %s\n", f.pat.newest)
|
||||||
return ignoreIfNoMatch, errIfNoMatch
|
return errIfNoMatch
|
||||||
}
|
}
|
||||||
|
|
||||||
debug.Log(" found match\n")
|
debug.Log(" found match\n")
|
||||||
f.out.PrintPattern(nodepath, node)
|
f.out.PrintPattern(nodepath, node)
|
||||||
return false, nil
|
return nil
|
||||||
})
|
}})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
|
func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
|
||||||
|
@ -346,17 +339,17 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
f.out.newsn = sn
|
f.out.newsn = sn
|
||||||
return walker.Walk(ctx, f.repo, *sn.Tree, f.ignoreTrees, func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
|
return walker.Walk(ctx, f.repo, *sn.Tree, walker.WalkVisitor{ProcessNode: func(parentTreeID restic.ID, nodepath string, node *restic.Node, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
debug.Log("Error loading tree %v: %v", parentTreeID, err)
|
debug.Log("Error loading tree %v: %v", parentTreeID, err)
|
||||||
|
|
||||||
Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
|
Printf("Unable to load tree %s\n ... which belongs to snapshot %s\n", parentTreeID, sn.ID())
|
||||||
|
|
||||||
return false, walker.ErrSkipNode
|
return walker.ErrSkipNode
|
||||||
}
|
}
|
||||||
|
|
||||||
if node == nil {
|
if node == nil {
|
||||||
return false, nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if node.Type == "dir" && f.treeIDs != nil {
|
if node.Type == "dir" && f.treeIDs != nil {
|
||||||
|
@ -374,7 +367,7 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
|
||||||
// looking for blobs)
|
// looking for blobs)
|
||||||
if f.itemsFound >= len(f.treeIDs) && f.blobIDs == nil {
|
if f.itemsFound >= len(f.treeIDs) && f.blobIDs == nil {
|
||||||
// Return an error to terminate the Walk
|
// Return an error to terminate the Walk
|
||||||
return true, errors.New("OK")
|
return errors.New("OK")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -395,8 +388,8 @@ func (f *Finder) findIDs(ctx context.Context, sn *restic.Snapshot) error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return false, nil
|
return nil
|
||||||
})
|
}})
|
||||||
}
|
}
|
||||||
|
|
||||||
var errAllPacksFound = errors.New("all packs found")
|
var errAllPacksFound = errors.New("all packs found")
|
||||||
|
@ -446,7 +439,10 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
|
||||||
|
|
||||||
if err != errAllPacksFound {
|
if err != errAllPacksFound {
|
||||||
// try to resolve unknown pack ids from the index
|
// try to resolve unknown pack ids from the index
|
||||||
packIDs = f.indexPacksToBlobs(ctx, packIDs)
|
packIDs, err = f.indexPacksToBlobs(ctx, packIDs)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(packIDs) > 0 {
|
if len(packIDs) > 0 {
|
||||||
|
@ -463,13 +459,13 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struct{}) map[string]struct{} {
|
func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struct{}) (map[string]struct{}, error) {
|
||||||
wctx, cancel := context.WithCancel(ctx)
|
wctx, cancel := context.WithCancel(ctx)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
|
||||||
// remember which packs were found in the index
|
// remember which packs were found in the index
|
||||||
indexPackIDs := make(map[string]struct{})
|
indexPackIDs := make(map[string]struct{})
|
||||||
f.repo.Index().Each(wctx, func(pb restic.PackedBlob) {
|
err := f.repo.ListBlobs(wctx, func(pb restic.PackedBlob) {
|
||||||
idStr := pb.PackID.String()
|
idStr := pb.PackID.String()
|
||||||
// keep entry in packIDs as Each() returns individual index entries
|
// keep entry in packIDs as Each() returns individual index entries
|
||||||
matchingID := false
|
matchingID := false
|
||||||
|
@ -488,6 +484,9 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
|
||||||
indexPackIDs[idStr] = struct{}{}
|
indexPackIDs[idStr] = struct{}{}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
for id := range indexPackIDs {
|
for id := range indexPackIDs {
|
||||||
delete(packIDs, id)
|
delete(packIDs, id)
|
||||||
|
@ -500,19 +499,17 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
|
||||||
}
|
}
|
||||||
Warnf("some pack files are missing from the repository, getting their blobs from the repository index: %v\n\n", list)
|
Warnf("some pack files are missing from the repository, getting their blobs from the repository index: %v\n\n", list)
|
||||||
}
|
}
|
||||||
return packIDs
|
return packIDs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *Finder) findObjectPack(id string, t restic.BlobType) {
|
func (f *Finder) findObjectPack(id string, t restic.BlobType) {
|
||||||
idx := f.repo.Index()
|
|
||||||
|
|
||||||
rid, err := restic.ParseID(id)
|
rid, err := restic.ParseID(id)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
Printf("Note: cannot find pack for object '%s', unable to parse ID: %v\n", id, err)
|
Printf("Note: cannot find pack for object '%s', unable to parse ID: %v\n", id, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
blobs := idx.Lookup(restic.BlobHandle{ID: rid, Type: t})
|
blobs := f.repo.LookupBlob(t, rid)
|
||||||
if len(blobs) == 0 {
|
if len(blobs) == 0 {
|
||||||
Printf("Object %s not found in the index\n", rid.Str())
|
Printf("Object %s not found in the index\n", rid.Str())
|
||||||
return
|
return
|
||||||
|
@ -570,21 +567,13 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
|
||||||
return errors.Fatal("cannot have several ID types")
|
return errors.Fatal("cannot have several ID types")
|
||||||
}
|
}
|
||||||
|
|
||||||
repo, err := OpenRepository(ctx, gopts)
|
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
defer unlock()
|
||||||
|
|
||||||
if !gopts.NoLock {
|
snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
|
||||||
var lock *restic.Lock
|
|
||||||
lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
|
|
||||||
defer unlockRepo(lock)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -594,10 +583,9 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
|
||||||
}
|
}
|
||||||
|
|
||||||
f := &Finder{
|
f := &Finder{
|
||||||
repo: repo,
|
repo: repo,
|
||||||
pat: pat,
|
pat: pat,
|
||||||
out: statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON},
|
out: statefulOutput{ListLong: opts.ListLong, HumanReadable: opts.HumanReadable, JSON: gopts.JSON},
|
||||||
ignoreTrees: restic.NewIDSet(),
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if opts.BlobID {
|
if opts.BlobID {
|
||||||
|
@ -624,6 +612,9 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
|
||||||
for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, opts.Snapshots) {
|
for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, opts.Snapshots) {
|
||||||
filteredSnapshots = append(filteredSnapshots, sn)
|
filteredSnapshots = append(filteredSnapshots, sn)
|
||||||
}
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
sort.Slice(filteredSnapshots, func(i, j int) bool {
|
sort.Slice(filteredSnapshots, func(i, j int) bool {
|
||||||
return filteredSnapshots[i].Time.Before(filteredSnapshots[j].Time)
|
return filteredSnapshots[i].Time.Before(filteredSnapshots[j].Time)
|
||||||
|
|
|
@ -3,11 +3,14 @@ package main
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
|
"github.com/restic/restic/internal/feature"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
|
"github.com/restic/restic/internal/ui/termstatus"
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -18,6 +21,9 @@ var cmdForget = &cobra.Command{
|
||||||
The "forget" command removes snapshots according to a policy. All snapshots are
|
The "forget" command removes snapshots according to a policy. All snapshots are
|
||||||
first divided into groups according to "--group-by", and after that the policy
|
first divided into groups according to "--group-by", and after that the policy
|
||||||
specified by the "--keep-*" options is applied to each group individually.
|
specified by the "--keep-*" options is applied to each group individually.
|
||||||
|
If there are not enough snapshots to keep one for each duration related
|
||||||
|
"--keep-{within-,}*" option, the oldest snapshot in the group is kept
|
||||||
|
additionally.
|
||||||
|
|
||||||
Please note that this command really only deletes the snapshot object in the
|
Please note that this command really only deletes the snapshot object in the
|
||||||
repository, which is a reference to data stored there. In order to remove the
|
repository, which is a reference to data stored there. In order to remove the
|
||||||
|
@ -33,7 +39,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
|
||||||
`,
|
`,
|
||||||
DisableAutoGenTag: true,
|
DisableAutoGenTag: true,
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
return runForget(cmd.Context(), forgetOptions, globalOptions, args)
|
term, cancel := setupTermstatus()
|
||||||
|
defer cancel()
|
||||||
|
return runForget(cmd.Context(), forgetOptions, forgetPruneOptions, globalOptions, term, args)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -88,6 +96,8 @@ type ForgetOptions struct {
|
||||||
WithinYearly restic.Duration
|
WithinYearly restic.Duration
|
||||||
KeepTags restic.TagLists
|
KeepTags restic.TagLists
|
||||||
|
|
||||||
|
UnsafeAllowRemoveAll bool
|
||||||
|
|
||||||
restic.SnapshotFilter
|
restic.SnapshotFilter
|
||||||
Compact bool
|
Compact bool
|
||||||
|
|
||||||
|
@ -98,6 +108,7 @@ type ForgetOptions struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
var forgetOptions ForgetOptions
|
var forgetOptions ForgetOptions
|
||||||
|
var forgetPruneOptions PruneOptions
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
cmdRoot.AddCommand(cmdForget)
|
cmdRoot.AddCommand(cmdForget)
|
||||||
|
@ -116,6 +127,7 @@ func init() {
|
||||||
f.VarP(&forgetOptions.WithinMonthly, "keep-within-monthly", "", "keep monthly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
|
f.VarP(&forgetOptions.WithinMonthly, "keep-within-monthly", "", "keep monthly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
|
||||||
f.VarP(&forgetOptions.WithinYearly, "keep-within-yearly", "", "keep yearly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
|
f.VarP(&forgetOptions.WithinYearly, "keep-within-yearly", "", "keep yearly snapshots that are newer than `duration` (eg. 1y5m7d2h) relative to the latest snapshot")
|
||||||
f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
|
f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)")
|
||||||
|
f.BoolVar(&forgetOptions.UnsafeAllowRemoveAll, "unsafe-allow-remove-all", false, "allow deleting all snapshots of a snapshot group")
|
||||||
|
|
||||||
initMultiSnapshotFilter(f, &forgetOptions.SnapshotFilter, false)
|
initMultiSnapshotFilter(f, &forgetOptions.SnapshotFilter, false)
|
||||||
f.StringArrayVar(&forgetOptions.Hosts, "hostname", nil, "only consider snapshots with the given `hostname` (can be specified multiple times)")
|
f.StringArrayVar(&forgetOptions.Hosts, "hostname", nil, "only consider snapshots with the given `hostname` (can be specified multiple times)")
|
||||||
|
@ -132,7 +144,7 @@ func init() {
|
||||||
f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
|
f.BoolVar(&forgetOptions.Prune, "prune", false, "automatically run the 'prune' command if snapshots have been removed")
|
||||||
|
|
||||||
f.SortFlags = false
|
f.SortFlags = false
|
||||||
addPruneOptions(cmdForget)
|
addPruneOptions(cmdForget, &forgetPruneOptions)
|
||||||
}
|
}
|
||||||
|
|
||||||
func verifyForgetOptions(opts *ForgetOptions) error {
|
func verifyForgetOptions(opts *ForgetOptions) error {
|
||||||
|
@ -151,7 +163,7 @@ func verifyForgetOptions(opts *ForgetOptions) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, args []string) error {
|
func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
|
||||||
err := verifyForgetOptions(&opts)
|
err := verifyForgetOptions(&opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -162,30 +174,31 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
repo, err := OpenRepository(ctx, gopts)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if gopts.NoLock && !opts.DryRun {
|
if gopts.NoLock && !opts.DryRun {
|
||||||
return errors.Fatal("--no-lock is only applicable in combination with --dry-run for forget command")
|
return errors.Fatal("--no-lock is only applicable in combination with --dry-run for forget command")
|
||||||
}
|
}
|
||||||
|
|
||||||
if !opts.DryRun || !gopts.NoLock {
|
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, opts.DryRun && gopts.NoLock)
|
||||||
var lock *restic.Lock
|
if err != nil {
|
||||||
lock, ctx, err = lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
|
return err
|
||||||
defer unlockRepo(lock)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
defer unlock()
|
||||||
|
|
||||||
|
verbosity := gopts.verbosity
|
||||||
|
if gopts.JSON {
|
||||||
|
verbosity = 0
|
||||||
|
}
|
||||||
|
printer := newTerminalProgressPrinter(verbosity, term)
|
||||||
|
|
||||||
var snapshots restic.Snapshots
|
var snapshots restic.Snapshots
|
||||||
removeSnIDs := restic.NewIDSet()
|
removeSnIDs := restic.NewIDSet()
|
||||||
|
|
||||||
for sn := range FindFilteredSnapshots(ctx, repo.Backend(), repo, &opts.SnapshotFilter, args) {
|
for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
|
||||||
snapshots = append(snapshots, sn)
|
snapshots = append(snapshots, sn)
|
||||||
}
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
var jsonGroups []*ForgetGroup
|
var jsonGroups []*ForgetGroup
|
||||||
|
|
||||||
|
@ -216,72 +229,87 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg
|
||||||
Tags: opts.KeepTags,
|
Tags: opts.KeepTags,
|
||||||
}
|
}
|
||||||
|
|
||||||
if policy.Empty() && len(args) == 0 {
|
if policy.Empty() {
|
||||||
if !gopts.JSON {
|
if opts.UnsafeAllowRemoveAll {
|
||||||
Verbosef("no policy was specified, no snapshots will be removed\n")
|
if opts.SnapshotFilter.Empty() {
|
||||||
|
return errors.Fatal("--unsafe-allow-remove-all is not allowed unless a snapshot filter option is specified")
|
||||||
|
}
|
||||||
|
// UnsafeAllowRemoveAll together with snapshot filter is fine
|
||||||
|
} else {
|
||||||
|
return errors.Fatal("no policy was specified, no snapshots will be removed")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if !policy.Empty() {
|
printer.P("Applying Policy: %v\n", policy)
|
||||||
if !gopts.JSON {
|
|
||||||
Verbosef("Applying Policy: %v\n", policy)
|
|
||||||
}
|
|
||||||
|
|
||||||
for k, snapshotGroup := range snapshotGroups {
|
for k, snapshotGroup := range snapshotGroups {
|
||||||
if gopts.Verbose >= 1 && !gopts.JSON {
|
if gopts.Verbose >= 1 && !gopts.JSON {
|
||||||
err = PrintSnapshotGroupHeader(globalOptions.stdout, k)
|
err = PrintSnapshotGroupHeader(globalOptions.stdout, k)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var key restic.SnapshotGroupKey
|
|
||||||
if json.Unmarshal([]byte(k), &key) != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
var fg ForgetGroup
|
var key restic.SnapshotGroupKey
|
||||||
fg.Tags = key.Tags
|
if json.Unmarshal([]byte(k), &key) != nil {
|
||||||
fg.Host = key.Hostname
|
return err
|
||||||
fg.Paths = key.Paths
|
}
|
||||||
|
|
||||||
keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy)
|
var fg ForgetGroup
|
||||||
|
fg.Tags = key.Tags
|
||||||
|
fg.Host = key.Hostname
|
||||||
|
fg.Paths = key.Paths
|
||||||
|
|
||||||
if len(keep) != 0 && !gopts.Quiet && !gopts.JSON {
|
keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy)
|
||||||
Printf("keep %d snapshots:\n", len(keep))
|
|
||||||
PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact)
|
|
||||||
Printf("\n")
|
|
||||||
}
|
|
||||||
addJSONSnapshots(&fg.Keep, keep)
|
|
||||||
|
|
||||||
if len(remove) != 0 && !gopts.Quiet && !gopts.JSON {
|
if feature.Flag.Enabled(feature.SafeForgetKeepTags) && !policy.Empty() && len(keep) == 0 {
|
||||||
Printf("remove %d snapshots:\n", len(remove))
|
return fmt.Errorf("refusing to delete last snapshot of snapshot group \"%v\"", key.String())
|
||||||
PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact)
|
}
|
||||||
Printf("\n")
|
if len(keep) != 0 && !gopts.Quiet && !gopts.JSON {
|
||||||
}
|
printer.P("keep %d snapshots:\n", len(keep))
|
||||||
addJSONSnapshots(&fg.Remove, remove)
|
PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact)
|
||||||
|
printer.P("\n")
|
||||||
|
}
|
||||||
|
fg.Keep = asJSONSnapshots(keep)
|
||||||
|
|
||||||
fg.Reasons = reasons
|
if len(remove) != 0 && !gopts.Quiet && !gopts.JSON {
|
||||||
|
printer.P("remove %d snapshots:\n", len(remove))
|
||||||
|
PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact)
|
||||||
|
printer.P("\n")
|
||||||
|
}
|
||||||
|
fg.Remove = asJSONSnapshots(remove)
|
||||||
|
|
||||||
jsonGroups = append(jsonGroups, &fg)
|
fg.Reasons = asJSONKeeps(reasons)
|
||||||
|
|
||||||
for _, sn := range remove {
|
jsonGroups = append(jsonGroups, &fg)
|
||||||
removeSnIDs.Insert(*sn.ID())
|
|
||||||
}
|
for _, sn := range remove {
|
||||||
|
removeSnIDs.Insert(*sn.ID())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return ctx.Err()
|
||||||
|
}
|
||||||
|
|
||||||
if len(removeSnIDs) > 0 {
|
if len(removeSnIDs) > 0 {
|
||||||
if !opts.DryRun {
|
if !opts.DryRun {
|
||||||
err := DeleteFilesChecked(ctx, gopts, repo, removeSnIDs, restic.SnapshotFile)
|
bar := printer.NewCounter("files deleted")
|
||||||
|
err := restic.ParallelRemove(ctx, repo, removeSnIDs, restic.SnapshotFile, func(id restic.ID, err error) error {
|
||||||
|
if err != nil {
|
||||||
|
printer.E("unable to remove %v/%v from the repository\n", restic.SnapshotFile, id)
|
||||||
|
} else {
|
||||||
|
printer.VV("removed %v/%v\n", restic.SnapshotFile, id)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}, bar)
|
||||||
|
bar.Done()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if !gopts.JSON {
|
printer.P("Would have removed the following snapshots:\n%v\n\n", removeSnIDs)
|
||||||
Printf("Would have removed the following snapshots:\n%v\n\n", removeSnIDs)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -293,15 +321,13 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(removeSnIDs) > 0 && opts.Prune {
|
if len(removeSnIDs) > 0 && opts.Prune {
|
||||||
if !gopts.JSON {
|
if opts.DryRun {
|
||||||
if opts.DryRun {
|
printer.P("%d snapshots would be removed, running prune dry run\n", len(removeSnIDs))
|
||||||
Verbosef("%d snapshots would be removed, running prune dry run\n", len(removeSnIDs))
|
} else {
|
||||||
} else {
|
printer.P("%d snapshots have been removed, running prune\n", len(removeSnIDs))
|
||||||
Verbosef("%d snapshots have been removed, running prune\n", len(removeSnIDs))
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
pruneOptions.DryRun = opts.DryRun
|
pruneOptions.DryRun = opts.DryRun
|
||||||
return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs)
|
return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs, term)
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
@ -309,23 +335,47 @@ func runForget(ctx context.Context, opts ForgetOptions, gopts GlobalOptions, arg
|
||||||
|
|
||||||
// ForgetGroup helps to print what is forgotten in JSON.
|
// ForgetGroup helps to print what is forgotten in JSON.
|
||||||
type ForgetGroup struct {
|
type ForgetGroup struct {
|
||||||
Tags []string `json:"tags"`
|
Tags []string `json:"tags"`
|
||||||
Host string `json:"host"`
|
Host string `json:"host"`
|
||||||
Paths []string `json:"paths"`
|
Paths []string `json:"paths"`
|
||||||
Keep []Snapshot `json:"keep"`
|
Keep []Snapshot `json:"keep"`
|
||||||
Remove []Snapshot `json:"remove"`
|
Remove []Snapshot `json:"remove"`
|
||||||
Reasons []restic.KeepReason `json:"reasons"`
|
Reasons []KeepReason `json:"reasons"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func addJSONSnapshots(js *[]Snapshot, list restic.Snapshots) {
|
func asJSONSnapshots(list restic.Snapshots) []Snapshot {
|
||||||
|
var resultList []Snapshot
|
||||||
for _, sn := range list {
|
for _, sn := range list {
|
||||||
k := Snapshot{
|
k := Snapshot{
|
||||||
Snapshot: sn,
|
Snapshot: sn,
|
||||||
ID: sn.ID(),
|
ID: sn.ID(),
|
||||||
ShortID: sn.ID().Str(),
|
ShortID: sn.ID().Str(),
|
||||||
}
|
}
|
||||||
*js = append(*js, k)
|
resultList = append(resultList, k)
|
||||||
}
|
}
|
||||||
|
return resultList
|
||||||
|
}
|
||||||
|
|
||||||
|
// KeepReason helps to print KeepReasons as JSON with Snapshots with their ID included.
|
||||||
|
type KeepReason struct {
|
||||||
|
Snapshot Snapshot `json:"snapshot"`
|
||||||
|
Matches []string `json:"matches"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func asJSONKeeps(list []restic.KeepReason) []KeepReason {
|
||||||
|
var resultList []KeepReason
|
||||||
|
for _, keep := range list {
|
||||||
|
k := KeepReason{
|
||||||
|
Snapshot: Snapshot{
|
||||||
|
Snapshot: keep.Snapshot,
|
||||||
|
ID: keep.Snapshot.ID(),
|
||||||
|
ShortID: keep.Snapshot.ID().Str(),
|
||||||
|
},
|
||||||
|
Matches: keep.Matches,
|
||||||
|
}
|
||||||
|
resultList = append(resultList, k)
|
||||||
|
}
|
||||||
|
return resultList
|
||||||
}
|
}
|
||||||
|
|
||||||
func printJSONForget(stdout io.Writer, forgets []*ForgetGroup) error {
|
func printJSONForget(stdout io.Writer, forgets []*ForgetGroup) error {
|
||||||
|
|
|
@ -2,12 +2,65 @@ package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/restic/restic/internal/restic"
|
||||||
rtest "github.com/restic/restic/internal/test"
|
rtest "github.com/restic/restic/internal/test"
|
||||||
|
"github.com/restic/restic/internal/ui/termstatus"
|
||||||
)
|
)
|
||||||
|
|
||||||
func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
|
func testRunForgetMayFail(gopts GlobalOptions, opts ForgetOptions, args ...string) error {
|
||||||
opts := ForgetOptions{}
|
pruneOpts := PruneOptions{
|
||||||
rtest.OK(t, runForget(context.TODO(), opts, gopts, args))
|
MaxUnused: "5%",
|
||||||
|
}
|
||||||
|
return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
|
||||||
|
return runForget(context.TODO(), opts, pruneOpts, gopts, term, args)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testRunForget(t testing.TB, gopts GlobalOptions, opts ForgetOptions, args ...string) {
|
||||||
|
rtest.OK(t, testRunForgetMayFail(gopts, opts, args...))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunForgetSafetyNet(t *testing.T) {
|
||||||
|
env, cleanup := withTestEnvironment(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
testSetupBackupData(t, env)
|
||||||
|
|
||||||
|
opts := BackupOptions{
|
||||||
|
Host: "example",
|
||||||
|
}
|
||||||
|
testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, opts, env.gopts)
|
||||||
|
testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9")}, opts, env.gopts)
|
||||||
|
testListSnapshots(t, env.gopts, 2)
|
||||||
|
|
||||||
|
// --keep-tags invalid
|
||||||
|
err := testRunForgetMayFail(env.gopts, ForgetOptions{
|
||||||
|
KeepTags: restic.TagLists{restic.TagList{"invalid"}},
|
||||||
|
GroupBy: restic.SnapshotGroupByOptions{Host: true, Path: true},
|
||||||
|
})
|
||||||
|
rtest.Assert(t, strings.Contains(err.Error(), `refusing to delete last snapshot of snapshot group "host example, path`), "wrong error message got %v", err)
|
||||||
|
|
||||||
|
// disallow `forget --unsafe-allow-remove-all`
|
||||||
|
err = testRunForgetMayFail(env.gopts, ForgetOptions{
|
||||||
|
UnsafeAllowRemoveAll: true,
|
||||||
|
})
|
||||||
|
rtest.Assert(t, strings.Contains(err.Error(), `--unsafe-allow-remove-all is not allowed unless a snapshot filter option is specified`), "wrong error message got %v", err)
|
||||||
|
|
||||||
|
// disallow `forget` without options
|
||||||
|
err = testRunForgetMayFail(env.gopts, ForgetOptions{})
|
||||||
|
rtest.Assert(t, strings.Contains(err.Error(), `no policy was specified, no snapshots will be removed`), "wrong error message got %v", err)
|
||||||
|
|
||||||
|
// `forget --host example --unsafe-allow-remmove-all` should work
|
||||||
|
testRunForget(t, env.gopts, ForgetOptions{
|
||||||
|
UnsafeAllowRemoveAll: true,
|
||||||
|
GroupBy: restic.SnapshotGroupByOptions{Host: true, Path: true},
|
||||||
|
SnapshotFilter: restic.SnapshotFilter{
|
||||||
|
Hosts: []string{opts.Host},
|
||||||
|
},
|
||||||
|
})
|
||||||
|
testListSnapshots(t, env.gopts, 0)
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,7 +21,9 @@ EXIT STATUS
|
||||||
Exit status is 0 if the command was successful, and non-zero if there was any error.
|
Exit status is 0 if the command was successful, and non-zero if there was any error.
|
||||||
`,
|
`,
|
||||||
DisableAutoGenTag: true,
|
DisableAutoGenTag: true,
|
||||||
RunE: runGenerate,
|
RunE: func(_ *cobra.Command, args []string) error {
|
||||||
|
return runGenerate(genOpts, args)
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
type generateOptions struct {
|
type generateOptions struct {
|
||||||
|
@ -90,48 +92,48 @@ func writePowerShellCompletion(file string) error {
|
||||||
return cmdRoot.GenPowerShellCompletionFile(file)
|
return cmdRoot.GenPowerShellCompletionFile(file)
|
||||||
}
|
}
|
||||||
|
|
||||||
func runGenerate(_ *cobra.Command, args []string) error {
|
func runGenerate(opts generateOptions, args []string) error {
|
||||||
if len(args) > 0 {
|
if len(args) > 0 {
|
||||||
return errors.Fatal("the generate command expects no arguments, only options - please see `restic help generate` for usage and flags")
|
return errors.Fatal("the generate command expects no arguments, only options - please see `restic help generate` for usage and flags")
|
||||||
}
|
}
|
||||||
|
|
||||||
if genOpts.ManDir != "" {
|
if opts.ManDir != "" {
|
||||||
err := writeManpages(genOpts.ManDir)
|
err := writeManpages(opts.ManDir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if genOpts.BashCompletionFile != "" {
|
if opts.BashCompletionFile != "" {
|
||||||
err := writeBashCompletion(genOpts.BashCompletionFile)
|
err := writeBashCompletion(opts.BashCompletionFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if genOpts.FishCompletionFile != "" {
|
if opts.FishCompletionFile != "" {
|
||||||
err := writeFishCompletion(genOpts.FishCompletionFile)
|
err := writeFishCompletion(opts.FishCompletionFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if genOpts.ZSHCompletionFile != "" {
|
if opts.ZSHCompletionFile != "" {
|
||||||
err := writeZSHCompletion(genOpts.ZSHCompletionFile)
|
err := writeZSHCompletion(opts.ZSHCompletionFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if genOpts.PowerShellCompletionFile != "" {
|
if opts.PowerShellCompletionFile != "" {
|
||||||
err := writePowerShellCompletion(genOpts.PowerShellCompletionFile)
|
err := writePowerShellCompletion(opts.PowerShellCompletionFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var empty generateOptions
|
var empty generateOptions
|
||||||
if genOpts == empty {
|
if opts == empty {
|
||||||
return errors.Fatal("nothing to do, please specify at least one output file/dir")
|
return errors.Fatal("nothing to do, please specify at least one output file/dir")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -80,7 +80,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
gopts.password, err = ReadPasswordTwice(gopts,
|
gopts.password, err = ReadPasswordTwice(ctx, gopts,
|
||||||
"enter password for new repository: ",
|
"enter password for new repository: ",
|
||||||
"enter password again: ")
|
"enter password again: ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -131,7 +131,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
|
||||||
|
|
||||||
func maybeReadChunkerPolynomial(ctx context.Context, opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) {
|
func maybeReadChunkerPolynomial(ctx context.Context, opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) {
|
||||||
if opts.CopyChunkerParameters {
|
if opts.CopyChunkerParameters {
|
||||||
otherGopts, _, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary")
|
otherGopts, _, err := fillSecondaryGlobalOpts(ctx, opts.secondaryRepoOptions, gopts, "secondary")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,265 +1,18 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
"github.com/restic/restic/internal/errors"
|
|
||||||
"github.com/restic/restic/internal/repository"
|
|
||||||
"github.com/restic/restic/internal/restic"
|
|
||||||
"github.com/restic/restic/internal/ui/table"
|
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
|
|
||||||
var cmdKey = &cobra.Command{
|
var cmdKey = &cobra.Command{
|
||||||
Use: "key [flags] [list|add|remove|passwd] [ID]",
|
Use: "key",
|
||||||
Short: "Manage keys (passwords)",
|
Short: "Manage keys (passwords)",
|
||||||
Long: `
|
Long: `
|
||||||
The "key" command manages keys (passwords) for accessing the repository.
|
The "key" command allows you to set multiple access keys or passwords
|
||||||
|
per repository.
|
||||||
EXIT STATUS
|
`,
|
||||||
===========
|
|
||||||
|
|
||||||
Exit status is 0 if the command was successful, and non-zero if there was any error.
|
|
||||||
`,
|
|
||||||
DisableAutoGenTag: true,
|
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
|
||||||
return runKey(cmd.Context(), globalOptions, args)
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
|
||||||
newPasswordFile string
|
|
||||||
keyUsername string
|
|
||||||
keyHostname string
|
|
||||||
)
|
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
cmdRoot.AddCommand(cmdKey)
|
cmdRoot.AddCommand(cmdKey)
|
||||||
|
|
||||||
flags := cmdKey.Flags()
|
|
||||||
flags.StringVarP(&newPasswordFile, "new-password-file", "", "", "`file` from which to read the new password")
|
|
||||||
flags.StringVarP(&keyUsername, "user", "", "", "the username for new keys")
|
|
||||||
flags.StringVarP(&keyHostname, "host", "", "", "the hostname for new keys")
|
|
||||||
}
|
|
||||||
|
|
||||||
func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions) error {
|
|
||||||
type keyInfo struct {
|
|
||||||
Current bool `json:"current"`
|
|
||||||
ID string `json:"id"`
|
|
||||||
UserName string `json:"userName"`
|
|
||||||
HostName string `json:"hostName"`
|
|
||||||
Created string `json:"created"`
|
|
||||||
}
|
|
||||||
|
|
||||||
var m sync.Mutex
|
|
||||||
var keys []keyInfo
|
|
||||||
|
|
||||||
err := restic.ParallelList(ctx, s.Backend(), restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, size int64) error {
|
|
||||||
k, err := repository.LoadKey(ctx, s, id)
|
|
||||||
if err != nil {
|
|
||||||
Warnf("LoadKey() failed: %v\n", err)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
key := keyInfo{
|
|
||||||
Current: id == s.KeyID(),
|
|
||||||
ID: id.Str(),
|
|
||||||
UserName: k.Username,
|
|
||||||
HostName: k.Hostname,
|
|
||||||
Created: k.Created.Local().Format(TimeFormat),
|
|
||||||
}
|
|
||||||
|
|
||||||
m.Lock()
|
|
||||||
defer m.Unlock()
|
|
||||||
keys = append(keys, key)
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
if gopts.JSON {
|
|
||||||
return json.NewEncoder(globalOptions.stdout).Encode(keys)
|
|
||||||
}
|
|
||||||
|
|
||||||
tab := table.New()
|
|
||||||
tab.AddColumn(" ID", "{{if .Current}}*{{else}} {{end}}{{ .ID }}")
|
|
||||||
tab.AddColumn("User", "{{ .UserName }}")
|
|
||||||
tab.AddColumn("Host", "{{ .HostName }}")
|
|
||||||
tab.AddColumn("Created", "{{ .Created }}")
|
|
||||||
|
|
||||||
for _, key := range keys {
|
|
||||||
tab.AddRow(key)
|
|
||||||
}
|
|
||||||
|
|
||||||
return tab.Write(globalOptions.stdout)
|
|
||||||
}
|
|
||||||
|
|
||||||
// testKeyNewPassword is used to set a new password during integration testing.
|
|
||||||
var testKeyNewPassword string
|
|
||||||
|
|
||||||
func getNewPassword(gopts GlobalOptions) (string, error) {
|
|
||||||
if testKeyNewPassword != "" {
|
|
||||||
return testKeyNewPassword, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if newPasswordFile != "" {
|
|
||||||
return loadPasswordFromFile(newPasswordFile)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Since we already have an open repository, temporary remove the password
|
|
||||||
// to prompt the user for the passwd.
|
|
||||||
newopts := gopts
|
|
||||||
newopts.password = ""
|
|
||||||
|
|
||||||
return ReadPasswordTwice(newopts,
|
|
||||||
"enter new password: ",
|
|
||||||
"enter password again: ")
|
|
||||||
}
|
|
||||||
|
|
||||||
func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error {
|
|
||||||
pw, err := getNewPassword(gopts)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
id, err := repository.AddKey(ctx, repo, pw, keyUsername, keyHostname, repo.Key())
|
|
||||||
if err != nil {
|
|
||||||
return errors.Fatalf("creating new key failed: %v\n", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
Verbosef("saved new key as %s\n", id)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func deleteKey(ctx context.Context, repo *repository.Repository, id restic.ID) error {
|
|
||||||
if id == repo.KeyID() {
|
|
||||||
return errors.Fatal("refusing to remove key currently used to access repository")
|
|
||||||
}
|
|
||||||
|
|
||||||
h := restic.Handle{Type: restic.KeyFile, Name: id.String()}
|
|
||||||
err := repo.Backend().Remove(ctx, h)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
Verbosef("removed key %v\n", id)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions) error {
|
|
||||||
pw, err := getNewPassword(gopts)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
id, err := repository.AddKey(ctx, repo, pw, "", "", repo.Key())
|
|
||||||
if err != nil {
|
|
||||||
return errors.Fatalf("creating new key failed: %v\n", err)
|
|
||||||
}
|
|
||||||
oldID := repo.KeyID()
|
|
||||||
|
|
||||||
err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
h := restic.Handle{Type: restic.KeyFile, Name: oldID.String()}
|
|
||||||
err = repo.Backend().Remove(ctx, h)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
Verbosef("saved new key as %s\n", id)
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error {
|
|
||||||
// Verify new key to make sure it really works. A broken key can render the
|
|
||||||
// whole repository inaccessible
|
|
||||||
err := repo.SearchKey(ctx, pw, 0, key.ID().String())
|
|
||||||
if err != nil {
|
|
||||||
// the key is invalid, try to remove it
|
|
||||||
h := restic.Handle{Type: restic.KeyFile, Name: key.ID().String()}
|
|
||||||
_ = repo.Backend().Remove(ctx, h)
|
|
||||||
return errors.Fatalf("failed to access repository with new key: %v", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func runKey(ctx context.Context, gopts GlobalOptions, args []string) error {
|
|
||||||
if len(args) < 1 || (args[0] == "remove" && len(args) != 2) || (args[0] != "remove" && len(args) != 1) {
|
|
||||||
return errors.Fatal("wrong number of arguments")
|
|
||||||
}
|
|
||||||
|
|
||||||
repo, err := OpenRepository(ctx, gopts)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
switch args[0] {
|
|
||||||
case "list":
|
|
||||||
if !gopts.NoLock {
|
|
||||||
var lock *restic.Lock
|
|
||||||
lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
|
|
||||||
defer unlockRepo(lock)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return listKeys(ctx, repo, gopts)
|
|
||||||
case "add":
|
|
||||||
lock, ctx, err := lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
|
|
||||||
defer unlockRepo(lock)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return addKey(ctx, repo, gopts)
|
|
||||||
case "remove":
|
|
||||||
lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
|
|
||||||
defer unlockRepo(lock)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
id, err := restic.Find(ctx, repo.Backend(), restic.KeyFile, args[1])
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return deleteKey(ctx, repo, id)
|
|
||||||
case "passwd":
|
|
||||||
lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
|
|
||||||
defer unlockRepo(lock)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return changePassword(ctx, repo, gopts)
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func loadPasswordFromFile(pwdFile string) (string, error) {
|
|
||||||
s, err := os.ReadFile(pwdFile)
|
|
||||||
if os.IsNotExist(err) {
|
|
||||||
return "", errors.Fatalf("%s does not exist", pwdFile)
|
|
||||||
}
|
|
||||||
return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile")
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,144 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/restic/restic/internal/errors"
|
||||||
|
"github.com/restic/restic/internal/repository"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"github.com/spf13/pflag"
|
||||||
|
)
|
||||||
|
|
||||||
|
var cmdKeyAdd = &cobra.Command{
|
||||||
|
Use: "add",
|
||||||
|
Short: "Add a new key (password) to the repository; returns the new key ID",
|
||||||
|
Long: `
|
||||||
|
The "add" sub-command creates a new key and validates the key. Returns the new key ID.
|
||||||
|
|
||||||
|
EXIT STATUS
|
||||||
|
===========
|
||||||
|
|
||||||
|
Exit status is 0 if the command is successful, and non-zero if there was any error.
|
||||||
|
`,
|
||||||
|
DisableAutoGenTag: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
type KeyAddOptions struct {
|
||||||
|
NewPasswordFile string
|
||||||
|
InsecureNoPassword bool
|
||||||
|
Username string
|
||||||
|
Hostname string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (opts *KeyAddOptions) Add(flags *pflag.FlagSet) {
|
||||||
|
flags.StringVarP(&opts.NewPasswordFile, "new-password-file", "", "", "`file` from which to read the new password")
|
||||||
|
flags.BoolVar(&opts.InsecureNoPassword, "new-insecure-no-password", false, "add an empty password for the repository (insecure)")
|
||||||
|
flags.StringVarP(&opts.Username, "user", "", "", "the username for new key")
|
||||||
|
flags.StringVarP(&opts.Hostname, "host", "", "", "the hostname for new key")
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
cmdKey.AddCommand(cmdKeyAdd)
|
||||||
|
|
||||||
|
var keyAddOpts KeyAddOptions
|
||||||
|
keyAddOpts.Add(cmdKeyAdd.Flags())
|
||||||
|
cmdKeyAdd.RunE = func(cmd *cobra.Command, args []string) error {
|
||||||
|
return runKeyAdd(cmd.Context(), globalOptions, keyAddOpts, args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runKeyAdd(ctx context.Context, gopts GlobalOptions, opts KeyAddOptions, args []string) error {
|
||||||
|
if len(args) > 0 {
|
||||||
|
return fmt.Errorf("the key add command expects no arguments, only options - please see `restic help key add` for usage and flags")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, repo, unlock, err := openWithAppendLock(ctx, gopts, false)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
|
|
||||||
|
return addKey(ctx, repo, gopts, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyAddOptions) error {
|
||||||
|
pw, err := getNewPassword(ctx, gopts, opts.NewPasswordFile, opts.InsecureNoPassword)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
id, err := repository.AddKey(ctx, repo, pw, opts.Username, opts.Hostname, repo.Key())
|
||||||
|
if err != nil {
|
||||||
|
return errors.Fatalf("creating new key failed: %v\n", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
Verbosef("saved new key with ID %s\n", id.ID())
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// testKeyNewPassword is used to set a new password during integration testing.
|
||||||
|
var testKeyNewPassword string
|
||||||
|
|
||||||
|
func getNewPassword(ctx context.Context, gopts GlobalOptions, newPasswordFile string, insecureNoPassword bool) (string, error) {
|
||||||
|
if testKeyNewPassword != "" {
|
||||||
|
return testKeyNewPassword, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if insecureNoPassword {
|
||||||
|
if newPasswordFile != "" {
|
||||||
|
return "", fmt.Errorf("only either --new-password-file or --new-insecure-no-password may be specified")
|
||||||
|
}
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if newPasswordFile != "" {
|
||||||
|
password, err := loadPasswordFromFile(newPasswordFile)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
if password == "" {
|
||||||
|
return "", fmt.Errorf("an empty password is not allowed by default. Pass the flag `--new-insecure-no-password` to restic to disable this check")
|
||||||
|
}
|
||||||
|
return password, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Since we already have an open repository, temporary remove the password
|
||||||
|
// to prompt the user for the passwd.
|
||||||
|
newopts := gopts
|
||||||
|
newopts.password = ""
|
||||||
|
// empty passwords are already handled above
|
||||||
|
newopts.InsecureNoPassword = false
|
||||||
|
|
||||||
|
return ReadPasswordTwice(ctx, newopts,
|
||||||
|
"enter new password: ",
|
||||||
|
"enter password again: ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func loadPasswordFromFile(pwdFile string) (string, error) {
|
||||||
|
s, err := os.ReadFile(pwdFile)
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return "", errors.Fatalf("%s does not exist", pwdFile)
|
||||||
|
}
|
||||||
|
return strings.TrimSpace(string(s)), errors.Wrap(err, "Readfile")
|
||||||
|
}
|
||||||
|
|
||||||
|
func switchToNewKeyAndRemoveIfBroken(ctx context.Context, repo *repository.Repository, key *repository.Key, pw string) error {
|
||||||
|
// Verify new key to make sure it really works. A broken key can render the
|
||||||
|
// whole repository inaccessible
|
||||||
|
err := repo.SearchKey(ctx, pw, 0, key.ID().String())
|
||||||
|
if err != nil {
|
||||||
|
// the key is invalid, try to remove it
|
||||||
|
_ = repository.RemoveKey(ctx, repo, key.ID())
|
||||||
|
return errors.Fatalf("failed to access repository with new key: %v", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
|
@ -3,17 +3,20 @@ package main
|
||||||
import (
|
import (
|
||||||
"bufio"
|
"bufio"
|
||||||
"context"
|
"context"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"github.com/restic/restic/internal/backend"
|
||||||
"github.com/restic/restic/internal/repository"
|
"github.com/restic/restic/internal/repository"
|
||||||
"github.com/restic/restic/internal/restic"
|
|
||||||
rtest "github.com/restic/restic/internal/test"
|
rtest "github.com/restic/restic/internal/test"
|
||||||
)
|
)
|
||||||
|
|
||||||
func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string {
|
func testRunKeyListOtherIDs(t testing.TB, gopts GlobalOptions) []string {
|
||||||
buf, err := withCaptureStdout(func() error {
|
buf, err := withCaptureStdout(func() error {
|
||||||
return runKey(context.TODO(), gopts, []string{"list"})
|
return runKeyList(context.TODO(), gopts, []string{})
|
||||||
})
|
})
|
||||||
rtest.OK(t, err)
|
rtest.OK(t, err)
|
||||||
|
|
||||||
|
@ -36,21 +39,20 @@ func testRunKeyAddNewKey(t testing.TB, newPassword string, gopts GlobalOptions)
|
||||||
testKeyNewPassword = ""
|
testKeyNewPassword = ""
|
||||||
}()
|
}()
|
||||||
|
|
||||||
rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"}))
|
rtest.OK(t, runKeyAdd(context.TODO(), gopts, KeyAddOptions{}, []string{}))
|
||||||
}
|
}
|
||||||
|
|
||||||
func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) {
|
func testRunKeyAddNewKeyUserHost(t testing.TB, gopts GlobalOptions) {
|
||||||
testKeyNewPassword = "john's geheimnis"
|
testKeyNewPassword = "john's geheimnis"
|
||||||
defer func() {
|
defer func() {
|
||||||
testKeyNewPassword = ""
|
testKeyNewPassword = ""
|
||||||
keyUsername = ""
|
|
||||||
keyHostname = ""
|
|
||||||
}()
|
}()
|
||||||
|
|
||||||
rtest.OK(t, cmdKey.Flags().Parse([]string{"--user=john", "--host=example.com"}))
|
|
||||||
|
|
||||||
t.Log("adding key for john@example.com")
|
t.Log("adding key for john@example.com")
|
||||||
rtest.OK(t, runKey(context.TODO(), gopts, []string{"add"}))
|
rtest.OK(t, runKeyAdd(context.TODO(), gopts, KeyAddOptions{
|
||||||
|
Username: "john",
|
||||||
|
Hostname: "example.com",
|
||||||
|
}, []string{}))
|
||||||
|
|
||||||
repo, err := OpenRepository(context.TODO(), gopts)
|
repo, err := OpenRepository(context.TODO(), gopts)
|
||||||
rtest.OK(t, err)
|
rtest.OK(t, err)
|
||||||
|
@ -67,13 +69,13 @@ func testRunKeyPasswd(t testing.TB, newPassword string, gopts GlobalOptions) {
|
||||||
testKeyNewPassword = ""
|
testKeyNewPassword = ""
|
||||||
}()
|
}()
|
||||||
|
|
||||||
rtest.OK(t, runKey(context.TODO(), gopts, []string{"passwd"}))
|
rtest.OK(t, runKeyPasswd(context.TODO(), gopts, KeyPasswdOptions{}, []string{}))
|
||||||
}
|
}
|
||||||
|
|
||||||
func testRunKeyRemove(t testing.TB, gopts GlobalOptions, IDs []string) {
|
func testRunKeyRemove(t testing.TB, gopts GlobalOptions, IDs []string) {
|
||||||
t.Logf("remove %d keys: %q\n", len(IDs), IDs)
|
t.Logf("remove %d keys: %q\n", len(IDs), IDs)
|
||||||
for _, id := range IDs {
|
for _, id := range IDs {
|
||||||
rtest.OK(t, runKey(context.TODO(), gopts, []string{"remove", id}))
|
rtest.OK(t, runKeyRemove(context.TODO(), gopts, []string{id}))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -103,18 +105,55 @@ func TestKeyAddRemove(t *testing.T) {
|
||||||
|
|
||||||
env.gopts.password = passwordList[len(passwordList)-1]
|
env.gopts.password = passwordList[len(passwordList)-1]
|
||||||
t.Logf("testing access with last password %q\n", env.gopts.password)
|
t.Logf("testing access with last password %q\n", env.gopts.password)
|
||||||
rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"}))
|
rtest.OK(t, runKeyList(context.TODO(), env.gopts, []string{}))
|
||||||
testRunCheck(t, env.gopts)
|
testRunCheck(t, env.gopts)
|
||||||
|
|
||||||
testRunKeyAddNewKeyUserHost(t, env.gopts)
|
testRunKeyAddNewKeyUserHost(t, env.gopts)
|
||||||
}
|
}
|
||||||
|
|
||||||
type emptySaveBackend struct {
|
func TestKeyAddInvalid(t *testing.T) {
|
||||||
restic.Backend
|
env, cleanup := withTestEnvironment(t)
|
||||||
|
defer cleanup()
|
||||||
|
testRunInit(t, env.gopts)
|
||||||
|
|
||||||
|
err := runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{
|
||||||
|
NewPasswordFile: "some-file",
|
||||||
|
InsecureNoPassword: true,
|
||||||
|
}, []string{})
|
||||||
|
rtest.Assert(t, strings.Contains(err.Error(), "only either"), "unexpected error message, got %q", err)
|
||||||
|
|
||||||
|
pwfile := filepath.Join(t.TempDir(), "pwfile")
|
||||||
|
rtest.OK(t, os.WriteFile(pwfile, []byte{}, 0o666))
|
||||||
|
|
||||||
|
err = runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{
|
||||||
|
NewPasswordFile: pwfile,
|
||||||
|
}, []string{})
|
||||||
|
rtest.Assert(t, strings.Contains(err.Error(), "an empty password is not allowed by default"), "unexpected error message, got %q", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *emptySaveBackend) Save(ctx context.Context, h restic.Handle, _ restic.RewindReader) error {
|
func TestKeyAddEmpty(t *testing.T) {
|
||||||
return b.Backend.Save(ctx, h, restic.NewByteReader([]byte{}, nil))
|
env, cleanup := withTestEnvironment(t)
|
||||||
|
// must list keys more than once
|
||||||
|
env.gopts.backendTestHook = nil
|
||||||
|
defer cleanup()
|
||||||
|
testRunInit(t, env.gopts)
|
||||||
|
|
||||||
|
rtest.OK(t, runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{
|
||||||
|
InsecureNoPassword: true,
|
||||||
|
}, []string{}))
|
||||||
|
|
||||||
|
env.gopts.password = ""
|
||||||
|
env.gopts.InsecureNoPassword = true
|
||||||
|
|
||||||
|
testRunCheck(t, env.gopts)
|
||||||
|
}
|
||||||
|
|
||||||
|
type emptySaveBackend struct {
|
||||||
|
backend.Backend
|
||||||
|
}
|
||||||
|
|
||||||
|
func (b *emptySaveBackend) Save(ctx context.Context, h backend.Handle, _ backend.RewindReader) error {
|
||||||
|
return b.Backend.Save(ctx, h, backend.NewByteReader([]byte{}, nil))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestKeyProblems(t *testing.T) {
|
func TestKeyProblems(t *testing.T) {
|
||||||
|
@ -122,7 +161,7 @@ func TestKeyProblems(t *testing.T) {
|
||||||
defer cleanup()
|
defer cleanup()
|
||||||
|
|
||||||
testRunInit(t, env.gopts)
|
testRunInit(t, env.gopts)
|
||||||
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) {
|
env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
|
||||||
return &emptySaveBackend{r}, nil
|
return &emptySaveBackend{r}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -131,15 +170,45 @@ func TestKeyProblems(t *testing.T) {
|
||||||
testKeyNewPassword = ""
|
testKeyNewPassword = ""
|
||||||
}()
|
}()
|
||||||
|
|
||||||
err := runKey(context.TODO(), env.gopts, []string{"passwd"})
|
err := runKeyPasswd(context.TODO(), env.gopts, KeyPasswdOptions{}, []string{})
|
||||||
t.Log(err)
|
t.Log(err)
|
||||||
rtest.Assert(t, err != nil, "expected passwd change to fail")
|
rtest.Assert(t, err != nil, "expected passwd change to fail")
|
||||||
|
|
||||||
err = runKey(context.TODO(), env.gopts, []string{"add"})
|
err = runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{}, []string{})
|
||||||
t.Log(err)
|
t.Log(err)
|
||||||
rtest.Assert(t, err != nil, "expected key adding to fail")
|
rtest.Assert(t, err != nil, "expected key adding to fail")
|
||||||
|
|
||||||
t.Logf("testing access with initial password %q\n", env.gopts.password)
|
t.Logf("testing access with initial password %q\n", env.gopts.password)
|
||||||
rtest.OK(t, runKey(context.TODO(), env.gopts, []string{"list"}))
|
rtest.OK(t, runKeyList(context.TODO(), env.gopts, []string{}))
|
||||||
testRunCheck(t, env.gopts)
|
testRunCheck(t, env.gopts)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestKeyCommandInvalidArguments(t *testing.T) {
|
||||||
|
env, cleanup := withTestEnvironment(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
testRunInit(t, env.gopts)
|
||||||
|
env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
|
||||||
|
return &emptySaveBackend{r}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
err := runKeyAdd(context.TODO(), env.gopts, KeyAddOptions{}, []string{"johndoe"})
|
||||||
|
t.Log(err)
|
||||||
|
rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key add: %v", err)
|
||||||
|
|
||||||
|
err = runKeyPasswd(context.TODO(), env.gopts, KeyPasswdOptions{}, []string{"johndoe"})
|
||||||
|
t.Log(err)
|
||||||
|
rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key passwd: %v", err)
|
||||||
|
|
||||||
|
err = runKeyList(context.TODO(), env.gopts, []string{"johndoe"})
|
||||||
|
t.Log(err)
|
||||||
|
rtest.Assert(t, err != nil && strings.Contains(err.Error(), "no arguments"), "unexpected error for key list: %v", err)
|
||||||
|
|
||||||
|
err = runKeyRemove(context.TODO(), env.gopts, []string{})
|
||||||
|
t.Log(err)
|
||||||
|
rtest.Assert(t, err != nil && strings.Contains(err.Error(), "one argument"), "unexpected error for key remove: %v", err)
|
||||||
|
|
||||||
|
err = runKeyRemove(context.TODO(), env.gopts, []string{"john", "doe"})
|
||||||
|
t.Log(err)
|
||||||
|
rtest.Assert(t, err != nil && strings.Contains(err.Error(), "one argument"), "unexpected error for key remove: %v", err)
|
||||||
|
}
|
||||||
|
|
|
@ -0,0 +1,106 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/restic/restic/internal/repository"
|
||||||
|
"github.com/restic/restic/internal/restic"
|
||||||
|
"github.com/restic/restic/internal/ui/table"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
var cmdKeyList = &cobra.Command{
|
||||||
|
Use: "list",
|
||||||
|
Short: "List keys (passwords)",
|
||||||
|
Long: `
|
||||||
|
The "list" sub-command lists all the keys (passwords) associated with the repository.
|
||||||
|
Returns the key ID, username, hostname, created time and if it's the current key being
|
||||||
|
used to access the repository.
|
||||||
|
|
||||||
|
EXIT STATUS
|
||||||
|
===========
|
||||||
|
|
||||||
|
Exit status is 0 if the command is successful, and non-zero if there was any error.
|
||||||
|
`,
|
||||||
|
DisableAutoGenTag: true,
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
return runKeyList(cmd.Context(), globalOptions, args)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
cmdKey.AddCommand(cmdKeyList)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runKeyList(ctx context.Context, gopts GlobalOptions, args []string) error {
|
||||||
|
if len(args) > 0 {
|
||||||
|
return fmt.Errorf("the key list command expects no arguments, only options - please see `restic help key list` for usage and flags")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
|
|
||||||
|
return listKeys(ctx, repo, gopts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func listKeys(ctx context.Context, s *repository.Repository, gopts GlobalOptions) error {
|
||||||
|
type keyInfo struct {
|
||||||
|
Current bool `json:"current"`
|
||||||
|
ID string `json:"id"`
|
||||||
|
ShortID string `json:"-"`
|
||||||
|
UserName string `json:"userName"`
|
||||||
|
HostName string `json:"hostName"`
|
||||||
|
Created string `json:"created"`
|
||||||
|
}
|
||||||
|
|
||||||
|
var m sync.Mutex
|
||||||
|
var keys []keyInfo
|
||||||
|
|
||||||
|
err := restic.ParallelList(ctx, s, restic.KeyFile, s.Connections(), func(ctx context.Context, id restic.ID, _ int64) error {
|
||||||
|
k, err := repository.LoadKey(ctx, s, id)
|
||||||
|
if err != nil {
|
||||||
|
Warnf("LoadKey() failed: %v\n", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
key := keyInfo{
|
||||||
|
Current: id == s.KeyID(),
|
||||||
|
ID: id.String(),
|
||||||
|
ShortID: id.Str(),
|
||||||
|
UserName: k.Username,
|
||||||
|
HostName: k.Hostname,
|
||||||
|
Created: k.Created.Local().Format(TimeFormat),
|
||||||
|
}
|
||||||
|
|
||||||
|
m.Lock()
|
||||||
|
defer m.Unlock()
|
||||||
|
keys = append(keys, key)
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if gopts.JSON {
|
||||||
|
return json.NewEncoder(globalOptions.stdout).Encode(keys)
|
||||||
|
}
|
||||||
|
|
||||||
|
tab := table.New()
|
||||||
|
tab.AddColumn(" ID", "{{if .Current}}*{{else}} {{end}}{{ .ShortID }}")
|
||||||
|
tab.AddColumn("User", "{{ .UserName }}")
|
||||||
|
tab.AddColumn("Host", "{{ .HostName }}")
|
||||||
|
tab.AddColumn("Created", "{{ .Created }}")
|
||||||
|
|
||||||
|
for _, key := range keys {
|
||||||
|
tab.AddRow(key)
|
||||||
|
}
|
||||||
|
|
||||||
|
return tab.Write(globalOptions.stdout)
|
||||||
|
}
|
|
@ -0,0 +1,80 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/restic/restic/internal/errors"
|
||||||
|
"github.com/restic/restic/internal/repository"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
var cmdKeyPasswd = &cobra.Command{
|
||||||
|
Use: "passwd",
|
||||||
|
Short: "Change key (password); creates a new key ID and removes the old key ID, returns new key ID",
|
||||||
|
Long: `
|
||||||
|
The "passwd" sub-command creates a new key, validates the key and remove the old key ID.
|
||||||
|
Returns the new key ID.
|
||||||
|
|
||||||
|
EXIT STATUS
|
||||||
|
===========
|
||||||
|
|
||||||
|
Exit status is 0 if the command is successful, and non-zero if there was any error.
|
||||||
|
`,
|
||||||
|
DisableAutoGenTag: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
type KeyPasswdOptions struct {
|
||||||
|
KeyAddOptions
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
cmdKey.AddCommand(cmdKeyPasswd)
|
||||||
|
|
||||||
|
var keyPasswdOpts KeyPasswdOptions
|
||||||
|
keyPasswdOpts.KeyAddOptions.Add(cmdKeyPasswd.Flags())
|
||||||
|
cmdKeyPasswd.RunE = func(cmd *cobra.Command, args []string) error {
|
||||||
|
return runKeyPasswd(cmd.Context(), globalOptions, keyPasswdOpts, args)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func runKeyPasswd(ctx context.Context, gopts GlobalOptions, opts KeyPasswdOptions, args []string) error {
|
||||||
|
if len(args) > 0 {
|
||||||
|
return fmt.Errorf("the key passwd command expects no arguments, only options - please see `restic help key passwd` for usage and flags")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
|
|
||||||
|
return changePassword(ctx, repo, gopts, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyPasswdOptions) error {
|
||||||
|
pw, err := getNewPassword(ctx, gopts, opts.NewPasswordFile, opts.InsecureNoPassword)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
id, err := repository.AddKey(ctx, repo, pw, "", "", repo.Key())
|
||||||
|
if err != nil {
|
||||||
|
return errors.Fatalf("creating new key failed: %v\n", err)
|
||||||
|
}
|
||||||
|
oldID := repo.KeyID()
|
||||||
|
|
||||||
|
err = switchToNewKeyAndRemoveIfBroken(ctx, repo, id, pw)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = repository.RemoveKey(ctx, repo, oldID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
Verbosef("saved new key as %s\n", id)
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
|
@ -0,0 +1,66 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/restic/restic/internal/errors"
|
||||||
|
"github.com/restic/restic/internal/repository"
|
||||||
|
"github.com/restic/restic/internal/restic"
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
)
|
||||||
|
|
||||||
|
var cmdKeyRemove = &cobra.Command{
|
||||||
|
Use: "remove [ID]",
|
||||||
|
Short: "Remove key ID (password) from the repository.",
|
||||||
|
Long: `
|
||||||
|
The "remove" sub-command removes the selected key ID. The "remove" command does not allow
|
||||||
|
removing the current key being used to access the repository.
|
||||||
|
|
||||||
|
EXIT STATUS
|
||||||
|
===========
|
||||||
|
|
||||||
|
Exit status is 0 if the command is successful, and non-zero if there was any error.
|
||||||
|
`,
|
||||||
|
DisableAutoGenTag: true,
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
return runKeyRemove(cmd.Context(), globalOptions, args)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
cmdKey.AddCommand(cmdKeyRemove)
|
||||||
|
}
|
||||||
|
|
||||||
|
func runKeyRemove(ctx context.Context, gopts GlobalOptions, args []string) error {
|
||||||
|
if len(args) != 1 {
|
||||||
|
return fmt.Errorf("key remove expects one argument as the key id")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer unlock()
|
||||||
|
|
||||||
|
return deleteKey(ctx, repo, args[0])
|
||||||
|
}
|
||||||
|
|
||||||
|
func deleteKey(ctx context.Context, repo *repository.Repository, idPrefix string) error {
|
||||||
|
id, err := restic.Find(ctx, repo, restic.KeyFile, idPrefix)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if id == repo.KeyID() {
|
||||||
|
return errors.Fatal("refusing to remove key currently used to access repository")
|
||||||
|
}
|
||||||
|
|
||||||
|
err = repository.RemoveKey(ctx, repo, id)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
Verbosef("removed key %v\n", id)
|
||||||
|
return nil
|
||||||
|
}
|
|
@ -4,7 +4,7 @@ import (
|
||||||
"context"
|
"context"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
"github.com/restic/restic/internal/index"
|
"github.com/restic/restic/internal/repository/index"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
@ -23,7 +23,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
|
||||||
`,
|
`,
|
||||||
DisableAutoGenTag: true,
|
DisableAutoGenTag: true,
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
return runList(cmd.Context(), cmd, globalOptions, args)
|
return runList(cmd.Context(), globalOptions, args)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -31,24 +31,16 @@ func init() {
|
||||||
cmdRoot.AddCommand(cmdList)
|
cmdRoot.AddCommand(cmdList)
|
||||||
}
|
}
|
||||||
|
|
||||||
func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args []string) error {
|
func runList(ctx context.Context, gopts GlobalOptions, args []string) error {
|
||||||
if len(args) != 1 {
|
if len(args) != 1 {
|
||||||
return errors.Fatal("type not specified, usage: " + cmd.Use)
|
return errors.Fatal("type not specified")
|
||||||
}
|
}
|
||||||
|
|
||||||
repo, err := OpenRepository(ctx, gopts)
|
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock || args[0] == "locks")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
defer unlock()
|
||||||
if !gopts.NoLock && args[0] != "locks" {
|
|
||||||
var lock *restic.Lock
|
|
||||||
lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
|
|
||||||
defer unlockRepo(lock)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
var t restic.FileType
|
var t restic.FileType
|
||||||
switch args[0] {
|
switch args[0] {
|
||||||
|
@ -63,20 +55,19 @@ func runList(ctx context.Context, cmd *cobra.Command, gopts GlobalOptions, args
|
||||||
case "locks":
|
case "locks":
|
||||||
t = restic.LockFile
|
t = restic.LockFile
|
||||||
case "blobs":
|
case "blobs":
|
||||||
return index.ForAllIndexes(ctx, repo.Backend(), repo, func(id restic.ID, idx *index.Index, oldFormat bool, err error) error {
|
return index.ForAllIndexes(ctx, repo, repo, func(_ restic.ID, idx *index.Index, _ bool, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
idx.Each(ctx, func(blobs restic.PackedBlob) {
|
return idx.Each(ctx, func(blobs restic.PackedBlob) {
|
||||||
Printf("%v %v\n", blobs.Type, blobs.ID)
|
Printf("%v %v\n", blobs.Type, blobs.ID)
|
||||||
})
|
})
|
||||||
return nil
|
|
||||||
})
|
})
|
||||||
default:
|
default:
|
||||||
return errors.Fatal("invalid type")
|
return errors.Fatal("invalid type")
|
||||||
}
|
}
|
||||||
|
|
||||||
return repo.List(ctx, t, func(id restic.ID, size int64) error {
|
return repo.List(ctx, t, func(id restic.ID, _ int64) error {
|
||||||
Printf("%s\n", id)
|
Printf("%s\n", id)
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
|
@ -12,7 +12,7 @@ import (
|
||||||
|
|
||||||
func testRunList(t testing.TB, tpe string, opts GlobalOptions) restic.IDs {
|
func testRunList(t testing.TB, tpe string, opts GlobalOptions) restic.IDs {
|
||||||
buf, err := withCaptureStdout(func() error {
|
buf, err := withCaptureStdout(func() error {
|
||||||
return runList(context.TODO(), cmdList, opts, []string{tpe})
|
return runList(context.TODO(), opts, []string{tpe})
|
||||||
})
|
})
|
||||||
rtest.OK(t, err)
|
rtest.OK(t, err)
|
||||||
return parseIDsFromReader(t, buf)
|
return parseIDsFromReader(t, buf)
|
||||||
|
|
|
@ -3,13 +3,14 @@ package main
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/backend"
|
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
"github.com/restic/restic/internal/fs"
|
"github.com/restic/restic/internal/fs"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
|
@ -52,6 +53,7 @@ type LsOptions struct {
|
||||||
restic.SnapshotFilter
|
restic.SnapshotFilter
|
||||||
Recursive bool
|
Recursive bool
|
||||||
HumanReadable bool
|
HumanReadable bool
|
||||||
|
Ncdu bool
|
||||||
}
|
}
|
||||||
|
|
||||||
var lsOptions LsOptions
|
var lsOptions LsOptions
|
||||||
|
@ -64,16 +66,49 @@ func init() {
|
||||||
flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
|
flags.BoolVarP(&lsOptions.ListLong, "long", "l", false, "use a long listing format showing size and mode")
|
||||||
flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
|
flags.BoolVar(&lsOptions.Recursive, "recursive", false, "include files in subfolders of the listed directories")
|
||||||
flags.BoolVar(&lsOptions.HumanReadable, "human-readable", false, "print sizes in human readable format")
|
flags.BoolVar(&lsOptions.HumanReadable, "human-readable", false, "print sizes in human readable format")
|
||||||
|
flags.BoolVar(&lsOptions.Ncdu, "ncdu", false, "output NCDU export format (pipe into 'ncdu -f -')")
|
||||||
}
|
}
|
||||||
|
|
||||||
type lsSnapshot struct {
|
type lsPrinter interface {
|
||||||
*restic.Snapshot
|
Snapshot(sn *restic.Snapshot)
|
||||||
ID *restic.ID `json:"id"`
|
Node(path string, node *restic.Node)
|
||||||
ShortID string `json:"short_id"`
|
LeaveDir(path string)
|
||||||
StructType string `json:"struct_type"` // "snapshot"
|
Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
type jsonLsPrinter struct {
|
||||||
|
enc *json.Encoder
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *jsonLsPrinter) Snapshot(sn *restic.Snapshot) {
|
||||||
|
type lsSnapshot struct {
|
||||||
|
*restic.Snapshot
|
||||||
|
ID *restic.ID `json:"id"`
|
||||||
|
ShortID string `json:"short_id"`
|
||||||
|
MessageType string `json:"message_type"` // "snapshot"
|
||||||
|
StructType string `json:"struct_type"` // "snapshot", deprecated
|
||||||
|
}
|
||||||
|
|
||||||
|
err := p.enc.Encode(lsSnapshot{
|
||||||
|
Snapshot: sn,
|
||||||
|
ID: sn.ID(),
|
||||||
|
ShortID: sn.ID().Str(),
|
||||||
|
MessageType: "snapshot",
|
||||||
|
StructType: "snapshot",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
Warnf("JSON encode failed: %v\n", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Print node in our custom JSON format, followed by a newline.
|
// Print node in our custom JSON format, followed by a newline.
|
||||||
|
func (p *jsonLsPrinter) Node(path string, node *restic.Node) {
|
||||||
|
err := lsNodeJSON(p.enc, path, node)
|
||||||
|
if err != nil {
|
||||||
|
Warnf("JSON encode failed: %v\n", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
|
func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
|
||||||
n := &struct {
|
n := &struct {
|
||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
|
@ -88,7 +123,8 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
|
||||||
AccessTime time.Time `json:"atime,omitempty"`
|
AccessTime time.Time `json:"atime,omitempty"`
|
||||||
ChangeTime time.Time `json:"ctime,omitempty"`
|
ChangeTime time.Time `json:"ctime,omitempty"`
|
||||||
Inode uint64 `json:"inode,omitempty"`
|
Inode uint64 `json:"inode,omitempty"`
|
||||||
StructType string `json:"struct_type"` // "node"
|
MessageType string `json:"message_type"` // "node"
|
||||||
|
StructType string `json:"struct_type"` // "node", deprecated
|
||||||
|
|
||||||
size uint64 // Target for Size pointer.
|
size uint64 // Target for Size pointer.
|
||||||
}{
|
}{
|
||||||
|
@ -104,6 +140,7 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
|
||||||
AccessTime: node.AccessTime,
|
AccessTime: node.AccessTime,
|
||||||
ChangeTime: node.ChangeTime,
|
ChangeTime: node.ChangeTime,
|
||||||
Inode: node.Inode,
|
Inode: node.Inode,
|
||||||
|
MessageType: "node",
|
||||||
StructType: "node",
|
StructType: "node",
|
||||||
}
|
}
|
||||||
// Always print size for regular files, even when empty,
|
// Always print size for regular files, even when empty,
|
||||||
|
@ -115,10 +152,117 @@ func lsNodeJSON(enc *json.Encoder, path string, node *restic.Node) error {
|
||||||
return enc.Encode(n)
|
return enc.Encode(n)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *jsonLsPrinter) LeaveDir(_ string) {}
|
||||||
|
func (p *jsonLsPrinter) Close() {}
|
||||||
|
|
||||||
|
type ncduLsPrinter struct {
|
||||||
|
out io.Writer
|
||||||
|
depth int
|
||||||
|
}
|
||||||
|
|
||||||
|
// lsSnapshotNcdu prints a restic snapshot in Ncdu save format.
|
||||||
|
// It opens the JSON list. Nodes are added with lsNodeNcdu and the list is closed by lsCloseNcdu.
|
||||||
|
// Format documentation: https://dev.yorhel.nl/ncdu/jsonfmt
|
||||||
|
func (p *ncduLsPrinter) Snapshot(sn *restic.Snapshot) {
|
||||||
|
const NcduMajorVer = 1
|
||||||
|
const NcduMinorVer = 2
|
||||||
|
|
||||||
|
snapshotBytes, err := json.Marshal(sn)
|
||||||
|
if err != nil {
|
||||||
|
Warnf("JSON encode failed: %v\n", err)
|
||||||
|
}
|
||||||
|
p.depth++
|
||||||
|
fmt.Fprintf(p.out, "[%d, %d, %s", NcduMajorVer, NcduMinorVer, string(snapshotBytes))
|
||||||
|
}
|
||||||
|
|
||||||
|
func lsNcduNode(_ string, node *restic.Node) ([]byte, error) {
|
||||||
|
type NcduNode struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Asize uint64 `json:"asize"`
|
||||||
|
Dsize uint64 `json:"dsize"`
|
||||||
|
Dev uint64 `json:"dev"`
|
||||||
|
Ino uint64 `json:"ino"`
|
||||||
|
NLink uint64 `json:"nlink"`
|
||||||
|
NotReg bool `json:"notreg"`
|
||||||
|
UID uint32 `json:"uid"`
|
||||||
|
GID uint32 `json:"gid"`
|
||||||
|
Mode uint16 `json:"mode"`
|
||||||
|
Mtime int64 `json:"mtime"`
|
||||||
|
}
|
||||||
|
|
||||||
|
outNode := NcduNode{
|
||||||
|
Name: node.Name,
|
||||||
|
Asize: node.Size,
|
||||||
|
Dsize: node.Size,
|
||||||
|
Dev: node.DeviceID,
|
||||||
|
Ino: node.Inode,
|
||||||
|
NLink: node.Links,
|
||||||
|
NotReg: node.Type != "dir" && node.Type != "file",
|
||||||
|
UID: node.UID,
|
||||||
|
GID: node.GID,
|
||||||
|
Mode: uint16(node.Mode & os.ModePerm),
|
||||||
|
Mtime: node.ModTime.Unix(),
|
||||||
|
}
|
||||||
|
// bits according to inode(7) manpage
|
||||||
|
if node.Mode&os.ModeSetuid != 0 {
|
||||||
|
outNode.Mode |= 0o4000
|
||||||
|
}
|
||||||
|
if node.Mode&os.ModeSetgid != 0 {
|
||||||
|
outNode.Mode |= 0o2000
|
||||||
|
}
|
||||||
|
if node.Mode&os.ModeSticky != 0 {
|
||||||
|
outNode.Mode |= 0o1000
|
||||||
|
}
|
||||||
|
|
||||||
|
return json.Marshal(outNode)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *ncduLsPrinter) Node(path string, node *restic.Node) {
|
||||||
|
out, err := lsNcduNode(path, node)
|
||||||
|
if err != nil {
|
||||||
|
Warnf("JSON encode failed: %v\n", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if node.Type == "dir" {
|
||||||
|
fmt.Fprintf(p.out, ",\n%s[\n%s%s", strings.Repeat(" ", p.depth), strings.Repeat(" ", p.depth+1), string(out))
|
||||||
|
p.depth++
|
||||||
|
} else {
|
||||||
|
fmt.Fprintf(p.out, ",\n%s%s", strings.Repeat(" ", p.depth), string(out))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *ncduLsPrinter) LeaveDir(_ string) {
|
||||||
|
p.depth--
|
||||||
|
fmt.Fprintf(p.out, "\n%s]", strings.Repeat(" ", p.depth))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *ncduLsPrinter) Close() {
|
||||||
|
fmt.Fprint(p.out, "\n]\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
type textLsPrinter struct {
|
||||||
|
dirs []string
|
||||||
|
ListLong bool
|
||||||
|
HumanReadable bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *textLsPrinter) Snapshot(sn *restic.Snapshot) {
|
||||||
|
Verbosef("%v filtered by %v:\n", sn, p.dirs)
|
||||||
|
}
|
||||||
|
func (p *textLsPrinter) Node(path string, node *restic.Node) {
|
||||||
|
Printf("%s\n", formatNode(path, node, p.ListLong, p.HumanReadable))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *textLsPrinter) LeaveDir(_ string) {}
|
||||||
|
func (p *textLsPrinter) Close() {}
|
||||||
|
|
||||||
func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error {
|
func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []string) error {
|
||||||
if len(args) == 0 {
|
if len(args) == 0 {
|
||||||
return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'")
|
return errors.Fatal("no snapshot ID specified, specify snapshot ID or use special ID 'latest'")
|
||||||
}
|
}
|
||||||
|
if opts.Ncdu && gopts.JSON {
|
||||||
|
return errors.Fatal("only either '--json' or '--ncdu' can be specified")
|
||||||
|
}
|
||||||
|
|
||||||
// extract any specific directories to walk
|
// extract any specific directories to walk
|
||||||
var dirs []string
|
var dirs []string
|
||||||
|
@ -165,12 +309,13 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
repo, err := OpenRepository(ctx, gopts)
|
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
defer unlock()
|
||||||
|
|
||||||
snapshotLister, err := backend.MemorizeList(ctx, repo.Backend(), restic.SnapshotFile)
|
snapshotLister, err := restic.MemorizeList(ctx, repo, restic.SnapshotFile)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -180,38 +325,21 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var printer lsPrinter
|
||||||
printSnapshot func(sn *restic.Snapshot)
|
|
||||||
printNode func(path string, node *restic.Node)
|
|
||||||
)
|
|
||||||
|
|
||||||
if gopts.JSON {
|
if gopts.JSON {
|
||||||
enc := json.NewEncoder(globalOptions.stdout)
|
printer = &jsonLsPrinter{
|
||||||
|
enc: json.NewEncoder(globalOptions.stdout),
|
||||||
printSnapshot = func(sn *restic.Snapshot) {
|
|
||||||
err := enc.Encode(lsSnapshot{
|
|
||||||
Snapshot: sn,
|
|
||||||
ID: sn.ID(),
|
|
||||||
ShortID: sn.ID().Str(),
|
|
||||||
StructType: "snapshot",
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
Warnf("JSON encode failed: %v\n", err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
} else if opts.Ncdu {
|
||||||
printNode = func(path string, node *restic.Node) {
|
printer = &ncduLsPrinter{
|
||||||
err := lsNodeJSON(enc, path, node)
|
out: globalOptions.stdout,
|
||||||
if err != nil {
|
|
||||||
Warnf("JSON encode failed: %v\n", err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
printSnapshot = func(sn *restic.Snapshot) {
|
printer = &textLsPrinter{
|
||||||
Verbosef("snapshot %s of %v filtered by %v at %s):\n", sn.ID().Str(), sn.Paths, dirs, sn.Time)
|
dirs: dirs,
|
||||||
}
|
ListLong: opts.ListLong,
|
||||||
printNode = func(path string, node *restic.Node) {
|
HumanReadable: opts.HumanReadable,
|
||||||
Printf("%s\n", formatNode(path, node, lsOptions.ListLong, lsOptions.HumanReadable))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -229,44 +357,55 @@ func runLs(ctx context.Context, opts LsOptions, gopts GlobalOptions, args []stri
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
printSnapshot(sn)
|
printer.Snapshot(sn)
|
||||||
|
|
||||||
err = walker.Walk(ctx, repo, *sn.Tree, nil, func(_ restic.ID, nodepath string, node *restic.Node, err error) (bool, error) {
|
processNode := func(_ restic.ID, nodepath string, node *restic.Node, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return err
|
||||||
}
|
}
|
||||||
if node == nil {
|
if node == nil {
|
||||||
return false, nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
if withinDir(nodepath) {
|
if withinDir(nodepath) {
|
||||||
// if we're within a dir, print the node
|
// if we're within a dir, print the node
|
||||||
printNode(nodepath, node)
|
printer.Node(nodepath, node)
|
||||||
|
|
||||||
// if recursive listing is requested, signal the walker that it
|
// if recursive listing is requested, signal the walker that it
|
||||||
// should continue walking recursively
|
// should continue walking recursively
|
||||||
if opts.Recursive {
|
if opts.Recursive {
|
||||||
return false, nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// if there's an upcoming match deeper in the tree (but we're not
|
// if there's an upcoming match deeper in the tree (but we're not
|
||||||
// there yet), signal the walker to descend into any subdirs
|
// there yet), signal the walker to descend into any subdirs
|
||||||
if approachingMatchingTree(nodepath) {
|
if approachingMatchingTree(nodepath) {
|
||||||
return false, nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// otherwise, signal the walker to not walk recursively into any
|
// otherwise, signal the walker to not walk recursively into any
|
||||||
// subdirs
|
// subdirs
|
||||||
if node.Type == "dir" {
|
if node.Type == "dir" {
|
||||||
return false, walker.ErrSkipNode
|
return walker.ErrSkipNode
|
||||||
}
|
}
|
||||||
return false, nil
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
err = walker.Walk(ctx, repo, *sn.Tree, walker.WalkVisitor{
|
||||||
|
ProcessNode: processNode,
|
||||||
|
LeaveDir: func(path string) {
|
||||||
|
// the root path `/` has no corresponding node and is thus also skipped by processNode
|
||||||
|
if withinDir(path) && path != "/" {
|
||||||
|
printer.LeaveDir(path)
|
||||||
|
}
|
||||||
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
printer.Close()
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,18 +2,46 @@ package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
rtest "github.com/restic/restic/internal/test"
|
rtest "github.com/restic/restic/internal/test"
|
||||||
)
|
)
|
||||||
|
|
||||||
func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
|
func testRunLsWithOpts(t testing.TB, gopts GlobalOptions, opts LsOptions, args []string) []byte {
|
||||||
buf, err := withCaptureStdout(func() error {
|
buf, err := withCaptureStdout(func() error {
|
||||||
gopts.Quiet = true
|
gopts.Quiet = true
|
||||||
opts := LsOptions{}
|
return runLs(context.TODO(), opts, gopts, args)
|
||||||
return runLs(context.TODO(), opts, gopts, []string{snapshotID})
|
|
||||||
})
|
})
|
||||||
rtest.OK(t, err)
|
rtest.OK(t, err)
|
||||||
return strings.Split(buf.String(), "\n")
|
return buf.Bytes()
|
||||||
|
}
|
||||||
|
|
||||||
|
func testRunLs(t testing.TB, gopts GlobalOptions, snapshotID string) []string {
|
||||||
|
out := testRunLsWithOpts(t, gopts, LsOptions{}, []string{snapshotID})
|
||||||
|
return strings.Split(string(out), "\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertIsValidJSON(t *testing.T, data []byte) {
|
||||||
|
// Sanity check: output must be valid JSON.
|
||||||
|
var v interface{}
|
||||||
|
err := json.Unmarshal(data, &v)
|
||||||
|
rtest.OK(t, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunLsNcdu(t *testing.T) {
|
||||||
|
env, cleanup := withTestEnvironment(t)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
testRunInit(t, env.gopts)
|
||||||
|
opts := BackupOptions{}
|
||||||
|
testRunBackup(t, filepath.Dir(env.testdata), []string{"testdata"}, opts, env.gopts)
|
||||||
|
|
||||||
|
ncdu := testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest"})
|
||||||
|
assertIsValidJSON(t, ncdu)
|
||||||
|
|
||||||
|
ncdu = testRunLsWithOpts(t, env.gopts, LsOptions{Ncdu: true}, []string{"latest", "/testdata"})
|
||||||
|
assertIsValidJSON(t, ncdu)
|
||||||
}
|
}
|
||||||
|
|
|
@ -11,78 +11,94 @@ import (
|
||||||
rtest "github.com/restic/restic/internal/test"
|
rtest "github.com/restic/restic/internal/test"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type lsTestNode struct {
|
||||||
|
path string
|
||||||
|
restic.Node
|
||||||
|
}
|
||||||
|
|
||||||
|
var lsTestNodes = []lsTestNode{
|
||||||
|
// Mode is omitted when zero.
|
||||||
|
// Permissions, by convention is "-" per mode bit
|
||||||
|
{
|
||||||
|
path: "/bar/baz",
|
||||||
|
Node: restic.Node{
|
||||||
|
Name: "baz",
|
||||||
|
Type: "file",
|
||||||
|
Size: 12345,
|
||||||
|
UID: 10000000,
|
||||||
|
GID: 20000000,
|
||||||
|
|
||||||
|
User: "nobody",
|
||||||
|
Group: "nobodies",
|
||||||
|
Links: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
// Even empty files get an explicit size.
|
||||||
|
{
|
||||||
|
path: "/foo/empty",
|
||||||
|
Node: restic.Node{
|
||||||
|
Name: "empty",
|
||||||
|
Type: "file",
|
||||||
|
Size: 0,
|
||||||
|
UID: 1001,
|
||||||
|
GID: 1001,
|
||||||
|
|
||||||
|
User: "not printed",
|
||||||
|
Group: "not printed",
|
||||||
|
Links: 0xF00,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
// Non-regular files do not get a size.
|
||||||
|
// Mode is printed in decimal, including the type bits.
|
||||||
|
{
|
||||||
|
path: "/foo/link",
|
||||||
|
Node: restic.Node{
|
||||||
|
Name: "link",
|
||||||
|
Type: "symlink",
|
||||||
|
Mode: os.ModeSymlink | 0777,
|
||||||
|
LinkTarget: "not printed",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
{
|
||||||
|
path: "/some/directory",
|
||||||
|
Node: restic.Node{
|
||||||
|
Name: "directory",
|
||||||
|
Type: "dir",
|
||||||
|
Mode: os.ModeDir | 0755,
|
||||||
|
ModTime: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC),
|
||||||
|
AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC),
|
||||||
|
ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
|
||||||
|
// Test encoding of setuid/setgid/sticky bit
|
||||||
|
{
|
||||||
|
path: "/some/sticky",
|
||||||
|
Node: restic.Node{
|
||||||
|
Name: "sticky",
|
||||||
|
Type: "dir",
|
||||||
|
Mode: os.ModeDir | 0755 | os.ModeSetuid | os.ModeSetgid | os.ModeSticky,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
func TestLsNodeJSON(t *testing.T) {
|
func TestLsNodeJSON(t *testing.T) {
|
||||||
for _, c := range []struct {
|
for i, expect := range []string{
|
||||||
path string
|
`{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","message_type":"node","struct_type":"node"}`,
|
||||||
restic.Node
|
`{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","message_type":"node","struct_type":"node"}`,
|
||||||
expect string
|
`{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","message_type":"node","struct_type":"node"}`,
|
||||||
}{
|
`{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","message_type":"node","struct_type":"node"}`,
|
||||||
// Mode is omitted when zero.
|
`{"name":"sticky","type":"dir","path":"/some/sticky","uid":0,"gid":0,"mode":2161115629,"permissions":"dugtrwxr-xr-x","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","message_type":"node","struct_type":"node"}`,
|
||||||
// Permissions, by convention is "-" per mode bit
|
|
||||||
{
|
|
||||||
path: "/bar/baz",
|
|
||||||
Node: restic.Node{
|
|
||||||
Name: "baz",
|
|
||||||
Type: "file",
|
|
||||||
Size: 12345,
|
|
||||||
UID: 10000000,
|
|
||||||
GID: 20000000,
|
|
||||||
|
|
||||||
User: "nobody",
|
|
||||||
Group: "nobodies",
|
|
||||||
Links: 1,
|
|
||||||
},
|
|
||||||
expect: `{"name":"baz","type":"file","path":"/bar/baz","uid":10000000,"gid":20000000,"size":12345,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
|
|
||||||
},
|
|
||||||
|
|
||||||
// Even empty files get an explicit size.
|
|
||||||
{
|
|
||||||
path: "/foo/empty",
|
|
||||||
Node: restic.Node{
|
|
||||||
Name: "empty",
|
|
||||||
Type: "file",
|
|
||||||
Size: 0,
|
|
||||||
UID: 1001,
|
|
||||||
GID: 1001,
|
|
||||||
|
|
||||||
User: "not printed",
|
|
||||||
Group: "not printed",
|
|
||||||
Links: 0xF00,
|
|
||||||
},
|
|
||||||
expect: `{"name":"empty","type":"file","path":"/foo/empty","uid":1001,"gid":1001,"size":0,"permissions":"----------","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
|
|
||||||
},
|
|
||||||
|
|
||||||
// Non-regular files do not get a size.
|
|
||||||
// Mode is printed in decimal, including the type bits.
|
|
||||||
{
|
|
||||||
path: "/foo/link",
|
|
||||||
Node: restic.Node{
|
|
||||||
Name: "link",
|
|
||||||
Type: "symlink",
|
|
||||||
Mode: os.ModeSymlink | 0777,
|
|
||||||
LinkTarget: "not printed",
|
|
||||||
},
|
|
||||||
expect: `{"name":"link","type":"symlink","path":"/foo/link","uid":0,"gid":0,"mode":134218239,"permissions":"Lrwxrwxrwx","mtime":"0001-01-01T00:00:00Z","atime":"0001-01-01T00:00:00Z","ctime":"0001-01-01T00:00:00Z","struct_type":"node"}`,
|
|
||||||
},
|
|
||||||
|
|
||||||
{
|
|
||||||
path: "/some/directory",
|
|
||||||
Node: restic.Node{
|
|
||||||
Name: "directory",
|
|
||||||
Type: "dir",
|
|
||||||
Mode: os.ModeDir | 0755,
|
|
||||||
ModTime: time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC),
|
|
||||||
AccessTime: time.Date(2021, 2, 3, 4, 5, 6, 7, time.UTC),
|
|
||||||
ChangeTime: time.Date(2022, 3, 4, 5, 6, 7, 8, time.UTC),
|
|
||||||
},
|
|
||||||
expect: `{"name":"directory","type":"dir","path":"/some/directory","uid":0,"gid":0,"mode":2147484141,"permissions":"drwxr-xr-x","mtime":"2020-01-02T03:04:05Z","atime":"2021-02-03T04:05:06.000000007Z","ctime":"2022-03-04T05:06:07.000000008Z","struct_type":"node"}`,
|
|
||||||
},
|
|
||||||
} {
|
} {
|
||||||
|
c := lsTestNodes[i]
|
||||||
buf := new(bytes.Buffer)
|
buf := new(bytes.Buffer)
|
||||||
enc := json.NewEncoder(buf)
|
enc := json.NewEncoder(buf)
|
||||||
err := lsNodeJSON(enc, c.path, &c.Node)
|
err := lsNodeJSON(enc, c.path, &c.Node)
|
||||||
rtest.OK(t, err)
|
rtest.OK(t, err)
|
||||||
rtest.Equals(t, c.expect+"\n", buf.String())
|
rtest.Equals(t, expect+"\n", buf.String())
|
||||||
|
|
||||||
// Sanity check: output must be valid JSON.
|
// Sanity check: output must be valid JSON.
|
||||||
var v interface{}
|
var v interface{}
|
||||||
|
@ -90,3 +106,54 @@ func TestLsNodeJSON(t *testing.T) {
|
||||||
rtest.OK(t, err)
|
rtest.OK(t, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLsNcduNode(t *testing.T) {
|
||||||
|
for i, expect := range []string{
|
||||||
|
`{"name":"baz","asize":12345,"dsize":12345,"dev":0,"ino":0,"nlink":1,"notreg":false,"uid":10000000,"gid":20000000,"mode":0,"mtime":-62135596800}`,
|
||||||
|
`{"name":"empty","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":3840,"notreg":false,"uid":1001,"gid":1001,"mode":0,"mtime":-62135596800}`,
|
||||||
|
`{"name":"link","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":true,"uid":0,"gid":0,"mode":511,"mtime":-62135596800}`,
|
||||||
|
`{"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":493,"mtime":1577934245}`,
|
||||||
|
`{"name":"sticky","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":4077,"mtime":-62135596800}`,
|
||||||
|
} {
|
||||||
|
c := lsTestNodes[i]
|
||||||
|
out, err := lsNcduNode(c.path, &c.Node)
|
||||||
|
rtest.OK(t, err)
|
||||||
|
rtest.Equals(t, expect, string(out))
|
||||||
|
|
||||||
|
// Sanity check: output must be valid JSON.
|
||||||
|
var v interface{}
|
||||||
|
err = json.Unmarshal(out, &v)
|
||||||
|
rtest.OK(t, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLsNcdu(t *testing.T) {
|
||||||
|
var buf bytes.Buffer
|
||||||
|
printer := &ncduLsPrinter{
|
||||||
|
out: &buf,
|
||||||
|
}
|
||||||
|
|
||||||
|
printer.Snapshot(&restic.Snapshot{
|
||||||
|
Hostname: "host",
|
||||||
|
Paths: []string{"/example"},
|
||||||
|
})
|
||||||
|
printer.Node("/directory", &restic.Node{
|
||||||
|
Type: "dir",
|
||||||
|
Name: "directory",
|
||||||
|
})
|
||||||
|
printer.Node("/directory/data", &restic.Node{
|
||||||
|
Type: "file",
|
||||||
|
Name: "data",
|
||||||
|
Size: 42,
|
||||||
|
})
|
||||||
|
printer.LeaveDir("/directory")
|
||||||
|
printer.Close()
|
||||||
|
|
||||||
|
rtest.Equals(t, `[1, 2, {"time":"0001-01-01T00:00:00Z","tree":null,"paths":["/example"],"hostname":"host"},
|
||||||
|
[
|
||||||
|
{"name":"directory","asize":0,"dsize":0,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800},
|
||||||
|
{"name":"data","asize":42,"dsize":42,"dev":0,"ino":0,"nlink":0,"notreg":false,"uid":0,"gid":0,"mode":0,"mtime":-62135596800}
|
||||||
|
]
|
||||||
|
]
|
||||||
|
`, buf.String())
|
||||||
|
}
|
||||||
|
|
|
@ -5,6 +5,8 @@ import (
|
||||||
|
|
||||||
"github.com/restic/restic/internal/migrations"
|
"github.com/restic/restic/internal/migrations"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
|
"github.com/restic/restic/internal/ui/progress"
|
||||||
|
"github.com/restic/restic/internal/ui/termstatus"
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
|
@ -24,7 +26,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
|
||||||
`,
|
`,
|
||||||
DisableAutoGenTag: true,
|
DisableAutoGenTag: true,
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
return runMigrate(cmd.Context(), migrateOptions, globalOptions, args)
|
term, cancel := setupTermstatus()
|
||||||
|
defer cancel()
|
||||||
|
return runMigrate(cmd.Context(), migrateOptions, globalOptions, args, term)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -41,8 +45,8 @@ func init() {
|
||||||
f.BoolVarP(&migrateOptions.Force, "force", "f", false, `apply a migration a second time`)
|
f.BoolVarP(&migrateOptions.Force, "force", "f", false, `apply a migration a second time`)
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkMigrations(ctx context.Context, repo restic.Repository) error {
|
func checkMigrations(ctx context.Context, repo restic.Repository, printer progress.Printer) error {
|
||||||
Printf("available migrations:\n")
|
printer.P("available migrations:\n")
|
||||||
found := false
|
found := false
|
||||||
|
|
||||||
for _, m := range migrations.All {
|
for _, m := range migrations.All {
|
||||||
|
@ -52,19 +56,19 @@ func checkMigrations(ctx context.Context, repo restic.Repository) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
if ok {
|
if ok {
|
||||||
Printf(" %v\t%v\n", m.Name(), m.Desc())
|
printer.P(" %v\t%v\n", m.Name(), m.Desc())
|
||||||
found = true
|
found = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if !found {
|
if !found {
|
||||||
Printf("no migrations found\n")
|
printer.P("no migrations found\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string) error {
|
func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, repo restic.Repository, args []string, term *termstatus.Terminal, printer progress.Printer) error {
|
||||||
var firsterr error
|
var firsterr error
|
||||||
for _, name := range args {
|
for _, name := range args {
|
||||||
for _, m := range migrations.All {
|
for _, m := range migrations.All {
|
||||||
|
@ -79,36 +83,37 @@ func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptio
|
||||||
if reason == "" {
|
if reason == "" {
|
||||||
reason = "check failed"
|
reason = "check failed"
|
||||||
}
|
}
|
||||||
Warnf("migration %v cannot be applied: %v\nIf you want to apply this migration anyway, re-run with option --force\n", m.Name(), reason)
|
printer.E("migration %v cannot be applied: %v\nIf you want to apply this migration anyway, re-run with option --force\n", m.Name(), reason)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
Warnf("check for migration %v failed, continuing anyway\n", m.Name())
|
printer.E("check for migration %v failed, continuing anyway\n", m.Name())
|
||||||
}
|
}
|
||||||
|
|
||||||
if m.RepoCheck() {
|
if m.RepoCheck() {
|
||||||
Printf("checking repository integrity...\n")
|
printer.P("checking repository integrity...\n")
|
||||||
|
|
||||||
checkOptions := CheckOptions{}
|
checkOptions := CheckOptions{}
|
||||||
checkGopts := gopts
|
checkGopts := gopts
|
||||||
// the repository is already locked
|
// the repository is already locked
|
||||||
checkGopts.NoLock = true
|
checkGopts.NoLock = true
|
||||||
err = runCheck(ctx, checkOptions, checkGopts, []string{})
|
|
||||||
|
err = runCheck(ctx, checkOptions, checkGopts, []string{}, term)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Printf("applying migration %v...\n", m.Name())
|
printer.P("applying migration %v...\n", m.Name())
|
||||||
if err = m.Apply(ctx, repo); err != nil {
|
if err = m.Apply(ctx, repo); err != nil {
|
||||||
Warnf("migration %v failed: %v\n", m.Name(), err)
|
printer.E("migration %v failed: %v\n", m.Name(), err)
|
||||||
if firsterr == nil {
|
if firsterr == nil {
|
||||||
firsterr = err
|
firsterr = err
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
Printf("migration %v: success\n", m.Name())
|
printer.P("migration %v: success\n", m.Name())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -116,21 +121,18 @@ func applyMigrations(ctx context.Context, opts MigrateOptions, gopts GlobalOptio
|
||||||
return firsterr
|
return firsterr
|
||||||
}
|
}
|
||||||
|
|
||||||
func runMigrate(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, args []string) error {
|
func runMigrate(ctx context.Context, opts MigrateOptions, gopts GlobalOptions, args []string, term *termstatus.Terminal) error {
|
||||||
repo, err := OpenRepository(ctx, gopts)
|
printer := newTerminalProgressPrinter(gopts.verbosity, term)
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
|
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
|
||||||
defer unlockRepo(lock)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
defer unlock()
|
||||||
|
|
||||||
if len(args) == 0 {
|
if len(args) == 0 {
|
||||||
return checkMigrations(ctx, repo)
|
return checkMigrations(ctx, repo, printer)
|
||||||
}
|
}
|
||||||
|
|
||||||
return applyMigrations(ctx, opts, gopts, repo, args)
|
return applyMigrations(ctx, opts, gopts, repo, args, term, printer)
|
||||||
}
|
}
|
||||||
|
|
|
@ -113,22 +113,23 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
|
||||||
return errors.Fatal("wrong number of parameters")
|
return errors.Fatal("wrong number of parameters")
|
||||||
}
|
}
|
||||||
|
|
||||||
debug.Log("start mount")
|
mountpoint := args[0]
|
||||||
defer debug.Log("finish mount")
|
|
||||||
|
|
||||||
repo, err := OpenRepository(ctx, gopts)
|
// Check the existence of the mount point at the earliest stage to
|
||||||
if err != nil {
|
// prevent unnecessary computations while opening the repository.
|
||||||
|
if _, err := resticfs.Stat(mountpoint); errors.Is(err, os.ErrNotExist) {
|
||||||
|
Verbosef("Mountpoint %s doesn't exist\n", mountpoint)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if !gopts.NoLock {
|
debug.Log("start mount")
|
||||||
var lock *restic.Lock
|
defer debug.Log("finish mount")
|
||||||
lock, ctx, err = lockRepo(ctx, repo, gopts.RetryLock, gopts.JSON)
|
|
||||||
defer unlockRepo(lock)
|
ctx, repo, unlock, err := openWithReadLock(ctx, gopts, gopts.NoLock)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
defer unlock()
|
||||||
|
|
||||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||||
err = repo.LoadIndex(ctx, bar)
|
err = repo.LoadIndex(ctx, bar)
|
||||||
|
@ -136,12 +137,6 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
mountpoint := args[0]
|
|
||||||
|
|
||||||
if _, err := resticfs.Stat(mountpoint); errors.Is(err, os.ErrNotExist) {
|
|
||||||
Verbosef("Mountpoint %s doesn't exist\n", mountpoint)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
mountOptions := []systemFuse.MountOption{
|
mountOptions := []systemFuse.MountOption{
|
||||||
systemFuse.ReadOnly(),
|
systemFuse.ReadOnly(),
|
||||||
systemFuse.FSName("restic"),
|
systemFuse.FSName("restic"),
|
||||||
|
@ -157,28 +152,15 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
AddCleanupHandler(func(code int) (int, error) {
|
systemFuse.Debug = func(msg interface{}) {
|
||||||
debug.Log("running umount cleanup handler for mount at %v", mountpoint)
|
debug.Log("fuse: %v", msg)
|
||||||
err := umount(mountpoint)
|
}
|
||||||
if err != nil {
|
|
||||||
Warnf("unable to umount (maybe already umounted or still in use?): %v\n", err)
|
|
||||||
}
|
|
||||||
// replace error code of sigint
|
|
||||||
if code == 130 {
|
|
||||||
code = 0
|
|
||||||
}
|
|
||||||
return code, nil
|
|
||||||
})
|
|
||||||
|
|
||||||
c, err := systemFuse.Mount(mountpoint, mountOptions...)
|
c, err := systemFuse.Mount(mountpoint, mountOptions...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
systemFuse.Debug = func(msg interface{}) {
|
|
||||||
debug.Log("fuse: %v", msg)
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg := fuse.Config{
|
cfg := fuse.Config{
|
||||||
OwnerIsRoot: opts.OwnerRoot,
|
OwnerIsRoot: opts.OwnerRoot,
|
||||||
Filter: opts.SnapshotFilter,
|
Filter: opts.SnapshotFilter,
|
||||||
|
@ -192,15 +174,26 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
|
||||||
Printf("When finished, quit with Ctrl-c here or umount the mountpoint.\n")
|
Printf("When finished, quit with Ctrl-c here or umount the mountpoint.\n")
|
||||||
|
|
||||||
debug.Log("serving mount at %v", mountpoint)
|
debug.Log("serving mount at %v", mountpoint)
|
||||||
err = fs.Serve(c, root)
|
|
||||||
if err != nil {
|
done := make(chan struct{})
|
||||||
return err
|
|
||||||
|
go func() {
|
||||||
|
defer close(done)
|
||||||
|
err = fs.Serve(c, root)
|
||||||
|
}()
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
debug.Log("running umount cleanup handler for mount at %v", mountpoint)
|
||||||
|
err := systemFuse.Unmount(mountpoint)
|
||||||
|
if err != nil {
|
||||||
|
Warnf("unable to umount (maybe already umounted or still in use?): %v\n", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return ErrOK
|
||||||
|
case <-done:
|
||||||
|
// clean shutdown, nothing to do
|
||||||
}
|
}
|
||||||
|
|
||||||
<-c.Ready
|
return err
|
||||||
return c.MountError
|
|
||||||
}
|
|
||||||
|
|
||||||
func umount(mountpoint string) error {
|
|
||||||
return systemFuse.Unmount(mountpoint)
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -12,8 +12,7 @@ import (
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/debug"
|
systemFuse "github.com/anacrolix/fuse"
|
||||||
"github.com/restic/restic/internal/repository"
|
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
rtest "github.com/restic/restic/internal/test"
|
rtest "github.com/restic/restic/internal/test"
|
||||||
)
|
)
|
||||||
|
@ -67,7 +66,7 @@ func testRunMount(t testing.TB, gopts GlobalOptions, dir string, wg *sync.WaitGr
|
||||||
func testRunUmount(t testing.TB, dir string) {
|
func testRunUmount(t testing.TB, dir string) {
|
||||||
var err error
|
var err error
|
||||||
for i := 0; i < mountWait; i++ {
|
for i := 0; i < mountWait; i++ {
|
||||||
if err = umount(dir); err == nil {
|
if err = systemFuse.Unmount(dir); err == nil {
|
||||||
t.Logf("directory %v umounted", dir)
|
t.Logf("directory %v umounted", dir)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -87,12 +86,12 @@ func listSnapshots(t testing.TB, dir string) []string {
|
||||||
return names
|
return names
|
||||||
}
|
}
|
||||||
|
|
||||||
func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Repository, mountpoint, repodir string, snapshotIDs restic.IDs, expectedSnapshotsInFuseDir int) {
|
func checkSnapshots(t testing.TB, gopts GlobalOptions, mountpoint string, snapshotIDs restic.IDs, expectedSnapshotsInFuseDir int) {
|
||||||
t.Logf("checking for %d snapshots: %v", len(snapshotIDs), snapshotIDs)
|
t.Logf("checking for %d snapshots: %v", len(snapshotIDs), snapshotIDs)
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go testRunMount(t, global, mountpoint, &wg)
|
go testRunMount(t, gopts, mountpoint, &wg)
|
||||||
waitForMount(t, mountpoint)
|
waitForMount(t, mountpoint)
|
||||||
defer wg.Wait()
|
defer wg.Wait()
|
||||||
defer testRunUmount(t, mountpoint)
|
defer testRunUmount(t, mountpoint)
|
||||||
|
@ -101,7 +100,7 @@ func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Reposit
|
||||||
t.Fatal(`virtual directory "snapshots" doesn't exist`)
|
t.Fatal(`virtual directory "snapshots" doesn't exist`)
|
||||||
}
|
}
|
||||||
|
|
||||||
ids := listSnapshots(t, repodir)
|
ids := listSnapshots(t, gopts.Repo)
|
||||||
t.Logf("found %v snapshots in repo: %v", len(ids), ids)
|
t.Logf("found %v snapshots in repo: %v", len(ids), ids)
|
||||||
|
|
||||||
namesInSnapshots := listSnapshots(t, mountpoint)
|
namesInSnapshots := listSnapshots(t, mountpoint)
|
||||||
|
@ -125,6 +124,10 @@ func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Reposit
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
_, repo, unlock, err := openWithReadLock(context.TODO(), gopts, false)
|
||||||
|
rtest.OK(t, err)
|
||||||
|
defer unlock()
|
||||||
|
|
||||||
for _, id := range snapshotIDs {
|
for _, id := range snapshotIDs {
|
||||||
snapshot, err := restic.LoadSnapshot(context.TODO(), repo, id)
|
snapshot, err := restic.LoadSnapshot(context.TODO(), repo, id)
|
||||||
rtest.OK(t, err)
|
rtest.OK(t, err)
|
||||||
|
@ -160,11 +163,6 @@ func TestMount(t *testing.T) {
|
||||||
t.Skip("Skipping fuse tests")
|
t.Skip("Skipping fuse tests")
|
||||||
}
|
}
|
||||||
|
|
||||||
debugEnabled := debug.TestLogToStderr(t)
|
|
||||||
if debugEnabled {
|
|
||||||
defer debug.TestDisableLog(t)
|
|
||||||
}
|
|
||||||
|
|
||||||
env, cleanup := withTestEnvironment(t)
|
env, cleanup := withTestEnvironment(t)
|
||||||
// must list snapshots more than once
|
// must list snapshots more than once
|
||||||
env.gopts.backendTestHook = nil
|
env.gopts.backendTestHook = nil
|
||||||
|
@ -172,10 +170,7 @@ func TestMount(t *testing.T) {
|
||||||
|
|
||||||
testRunInit(t, env.gopts)
|
testRunInit(t, env.gopts)
|
||||||
|
|
||||||
repo, err := OpenRepository(context.TODO(), env.gopts)
|
checkSnapshots(t, env.gopts, env.mountpoint, []restic.ID{}, 0)
|
||||||
rtest.OK(t, err)
|
|
||||||
|
|
||||||
checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, []restic.ID{}, 0)
|
|
||||||
|
|
||||||
rtest.SetupTarTestFixture(t, env.testdata, filepath.Join("testdata", "backup-data.tar.gz"))
|
rtest.SetupTarTestFixture(t, env.testdata, filepath.Join("testdata", "backup-data.tar.gz"))
|
||||||
|
|
||||||
|
@ -185,7 +180,7 @@ func TestMount(t *testing.T) {
|
||||||
rtest.Assert(t, len(snapshotIDs) == 1,
|
rtest.Assert(t, len(snapshotIDs) == 1,
|
||||||
"expected one snapshot, got %v", snapshotIDs)
|
"expected one snapshot, got %v", snapshotIDs)
|
||||||
|
|
||||||
checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, snapshotIDs, 2)
|
checkSnapshots(t, env.gopts, env.mountpoint, snapshotIDs, 2)
|
||||||
|
|
||||||
// second backup, implicit incremental
|
// second backup, implicit incremental
|
||||||
testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts)
|
testRunBackup(t, "", []string{env.testdata}, BackupOptions{}, env.gopts)
|
||||||
|
@ -193,7 +188,7 @@ func TestMount(t *testing.T) {
|
||||||
rtest.Assert(t, len(snapshotIDs) == 2,
|
rtest.Assert(t, len(snapshotIDs) == 2,
|
||||||
"expected two snapshots, got %v", snapshotIDs)
|
"expected two snapshots, got %v", snapshotIDs)
|
||||||
|
|
||||||
checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, snapshotIDs, 3)
|
checkSnapshots(t, env.gopts, env.mountpoint, snapshotIDs, 3)
|
||||||
|
|
||||||
// third backup, explicit incremental
|
// third backup, explicit incremental
|
||||||
bopts := BackupOptions{Parent: snapshotIDs[0].String()}
|
bopts := BackupOptions{Parent: snapshotIDs[0].String()}
|
||||||
|
@ -202,7 +197,7 @@ func TestMount(t *testing.T) {
|
||||||
rtest.Assert(t, len(snapshotIDs) == 3,
|
rtest.Assert(t, len(snapshotIDs) == 3,
|
||||||
"expected three snapshots, got %v", snapshotIDs)
|
"expected three snapshots, got %v", snapshotIDs)
|
||||||
|
|
||||||
checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, snapshotIDs, 4)
|
checkSnapshots(t, env.gopts, env.mountpoint, snapshotIDs, 4)
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMountSameTimestamps(t *testing.T) {
|
func TestMountSameTimestamps(t *testing.T) {
|
||||||
|
@ -217,14 +212,11 @@ func TestMountSameTimestamps(t *testing.T) {
|
||||||
|
|
||||||
rtest.SetupTarTestFixture(t, env.base, filepath.Join("testdata", "repo-same-timestamps.tar.gz"))
|
rtest.SetupTarTestFixture(t, env.base, filepath.Join("testdata", "repo-same-timestamps.tar.gz"))
|
||||||
|
|
||||||
repo, err := OpenRepository(context.TODO(), env.gopts)
|
|
||||||
rtest.OK(t, err)
|
|
||||||
|
|
||||||
ids := []restic.ID{
|
ids := []restic.ID{
|
||||||
restic.TestParseID("280303689e5027328889a06d718b729e96a1ce6ae9ef8290bff550459ae611ee"),
|
restic.TestParseID("280303689e5027328889a06d718b729e96a1ce6ae9ef8290bff550459ae611ee"),
|
||||||
restic.TestParseID("75ad6cdc0868e082f2596d5ab8705e9f7d87316f5bf5690385eeff8dbe49d9f5"),
|
restic.TestParseID("75ad6cdc0868e082f2596d5ab8705e9f7d87316f5bf5690385eeff8dbe49d9f5"),
|
||||||
restic.TestParseID("5fd0d8b2ef0fa5d23e58f1e460188abb0f525c0f0c4af8365a1280c807a80a1b"),
|
restic.TestParseID("5fd0d8b2ef0fa5d23e58f1e460188abb0f525c0f0c4af8365a1280c807a80a1b"),
|
||||||
}
|
}
|
||||||
|
|
||||||
checkSnapshots(t, env.gopts, repo, env.mountpoint, env.repo, ids, 4)
|
checkSnapshots(t, env.gopts, env.mountpoint, ids, 4)
|
||||||
}
|
}
|
||||||
|
|
|
@ -21,7 +21,7 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
|
||||||
`,
|
`,
|
||||||
Hidden: true,
|
Hidden: true,
|
||||||
DisableAutoGenTag: true,
|
DisableAutoGenTag: true,
|
||||||
Run: func(cmd *cobra.Command, args []string) {
|
Run: func(_ *cobra.Command, _ []string) {
|
||||||
fmt.Printf("All Extended Options:\n")
|
fmt.Printf("All Extended Options:\n")
|
||||||
var maxLen int
|
var maxLen int
|
||||||
for _, opt := range options.List() {
|
for _, opt := range options.List() {
|
||||||
|
|
|
@ -4,25 +4,20 @@ import (
|
||||||
"context"
|
"context"
|
||||||
"math"
|
"math"
|
||||||
"runtime"
|
"runtime"
|
||||||
"sort"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/debug"
|
"github.com/restic/restic/internal/debug"
|
||||||
"github.com/restic/restic/internal/errors"
|
"github.com/restic/restic/internal/errors"
|
||||||
"github.com/restic/restic/internal/index"
|
|
||||||
"github.com/restic/restic/internal/pack"
|
|
||||||
"github.com/restic/restic/internal/repository"
|
"github.com/restic/restic/internal/repository"
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/restic"
|
||||||
"github.com/restic/restic/internal/ui"
|
"github.com/restic/restic/internal/ui"
|
||||||
|
"github.com/restic/restic/internal/ui/progress"
|
||||||
|
"github.com/restic/restic/internal/ui/termstatus"
|
||||||
|
|
||||||
"github.com/spf13/cobra"
|
"github.com/spf13/cobra"
|
||||||
)
|
)
|
||||||
|
|
||||||
var errorIndexIncomplete = errors.Fatal("index is not complete")
|
|
||||||
var errorPacksMissing = errors.Fatal("packs from index missing in repo")
|
|
||||||
var errorSizeNotMatching = errors.Fatal("pack size does not match calculated size from index")
|
|
||||||
|
|
||||||
var cmdPrune = &cobra.Command{
|
var cmdPrune = &cobra.Command{
|
||||||
Use: "prune [flags]",
|
Use: "prune [flags]",
|
||||||
Short: "Remove unneeded data from the repository",
|
Short: "Remove unneeded data from the repository",
|
||||||
|
@ -36,8 +31,10 @@ EXIT STATUS
|
||||||
Exit status is 0 if the command was successful, and non-zero if there was any error.
|
Exit status is 0 if the command was successful, and non-zero if there was any error.
|
||||||
`,
|
`,
|
||||||
DisableAutoGenTag: true,
|
DisableAutoGenTag: true,
|
||||||
RunE: func(cmd *cobra.Command, args []string) error {
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||||
return runPrune(cmd.Context(), pruneOptions, globalOptions)
|
term, cancel := setupTermstatus()
|
||||||
|
defer cancel()
|
||||||
|
return runPrune(cmd.Context(), pruneOptions, globalOptions, term)
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -66,10 +63,10 @@ func init() {
|
||||||
f := cmdPrune.Flags()
|
f := cmdPrune.Flags()
|
||||||
f.BoolVarP(&pruneOptions.DryRun, "dry-run", "n", false, "do not modify the repository, just print what would be done")
|
f.BoolVarP(&pruneOptions.DryRun, "dry-run", "n", false, "do not modify the repository, just print what would be done")
|
||||||
f.StringVarP(&pruneOptions.UnsafeNoSpaceRecovery, "unsafe-recover-no-free-space", "", "", "UNSAFE, READ THE DOCUMENTATION BEFORE USING! Try to recover a repository stuck with no free space. Do not use without trying out 'prune --max-repack-size 0' first.")
|
f.StringVarP(&pruneOptions.UnsafeNoSpaceRecovery, "unsafe-recover-no-free-space", "", "", "UNSAFE, READ THE DOCUMENTATION BEFORE USING! Try to recover a repository stuck with no free space. Do not use without trying out 'prune --max-repack-size 0' first.")
|
||||||
addPruneOptions(cmdPrune)
|
addPruneOptions(cmdPrune, &pruneOptions)
|
||||||
}
|
}
|
||||||
|
|
||||||
func addPruneOptions(c *cobra.Command) {
|
func addPruneOptions(c *cobra.Command, pruneOptions *PruneOptions) {
|
||||||
f := c.Flags()
|
f := c.Flags()
|
||||||
f.StringVar(&pruneOptions.MaxUnused, "max-unused", "5%", "tolerate given `limit` of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited')")
|
f.StringVar(&pruneOptions.MaxUnused, "max-unused", "5%", "tolerate given `limit` of unused data (absolute value in bytes with suffixes k/K, m/M, g/G, t/T, a value in % or the word 'unlimited')")
|
||||||
f.StringVar(&pruneOptions.MaxRepackSize, "max-repack-size", "", "maximum `size` to repack (allowed suffixes: k/K, m/M, g/G, t/T)")
|
f.StringVar(&pruneOptions.MaxRepackSize, "max-repack-size", "", "maximum `size` to repack (allowed suffixes: k/K, m/M, g/G, t/T)")
|
||||||
|
@ -100,7 +97,7 @@ func verifyPruneOptions(opts *PruneOptions) error {
|
||||||
// parse MaxUnused either as unlimited, a percentage, or an absolute number of bytes
|
// parse MaxUnused either as unlimited, a percentage, or an absolute number of bytes
|
||||||
switch {
|
switch {
|
||||||
case maxUnused == "unlimited":
|
case maxUnused == "unlimited":
|
||||||
opts.maxUnusedBytes = func(used uint64) uint64 {
|
opts.maxUnusedBytes = func(_ uint64) uint64 {
|
||||||
return math.MaxUint64
|
return math.MaxUint64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -129,7 +126,7 @@ func verifyPruneOptions(opts *PruneOptions) error {
|
||||||
return errors.Fatalf("invalid number of bytes %q for --max-unused: %v", opts.MaxUnused, err)
|
return errors.Fatalf("invalid number of bytes %q for --max-unused: %v", opts.MaxUnused, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
opts.maxUnusedBytes = func(used uint64) uint64 {
|
opts.maxUnusedBytes = func(_ uint64) uint64 {
|
||||||
return uint64(size)
|
return uint64(size)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -137,7 +134,7 @@ func verifyPruneOptions(opts *PruneOptions) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error {
|
func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, term *termstatus.Terminal) error {
|
||||||
err := verifyPruneOptions(&opts)
|
err := verifyPruneOptions(&opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -147,18 +144,11 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error
|
||||||
return errors.Fatal("disabled compression and `--repack-uncompressed` are mutually exclusive")
|
return errors.Fatal("disabled compression and `--repack-uncompressed` are mutually exclusive")
|
||||||
}
|
}
|
||||||
|
|
||||||
repo, err := OpenRepository(ctx, gopts)
|
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
defer unlock()
|
||||||
if repo.Backend().Connections() < 2 {
|
|
||||||
return errors.Fatal("prune requires a backend connection limit of at least two")
|
|
||||||
}
|
|
||||||
|
|
||||||
if repo.Config().Version < 2 && opts.RepackUncompressed {
|
|
||||||
return errors.Fatal("compression requires at least repository format version 2")
|
|
||||||
}
|
|
||||||
|
|
||||||
if opts.UnsafeNoSpaceRecovery != "" {
|
if opts.UnsafeNoSpaceRecovery != "" {
|
||||||
repoID := repo.Config().ID
|
repoID := repo.Config().ID
|
||||||
|
@ -168,41 +158,51 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error
|
||||||
opts.unsafeRecovery = true
|
opts.unsafeRecovery = true
|
||||||
}
|
}
|
||||||
|
|
||||||
lock, ctx, err := lockRepoExclusive(ctx, repo, gopts.RetryLock, gopts.JSON)
|
return runPruneWithRepo(ctx, opts, gopts, repo, restic.NewIDSet(), term)
|
||||||
defer unlockRepo(lock)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return runPruneWithRepo(ctx, opts, gopts, repo, restic.NewIDSet())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet) error {
|
func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet, term *termstatus.Terminal) error {
|
||||||
// we do not need index updates while pruning!
|
|
||||||
repo.DisableAutoIndexUpdate()
|
|
||||||
|
|
||||||
if repo.Cache == nil {
|
if repo.Cache == nil {
|
||||||
Print("warning: running prune without a cache, this may be very slow!\n")
|
Print("warning: running prune without a cache, this may be very slow!\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
Verbosef("loading indexes...\n")
|
printer := newTerminalProgressPrinter(gopts.verbosity, term)
|
||||||
|
|
||||||
|
printer.P("loading indexes...\n")
|
||||||
// loading the index before the snapshots is ok, as we use an exclusive lock here
|
// loading the index before the snapshots is ok, as we use an exclusive lock here
|
||||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
|
||||||
err := repo.LoadIndex(ctx, bar)
|
err := repo.LoadIndex(ctx, bar)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
plan, stats, err := planPrune(ctx, opts, repo, ignoreSnapshots, gopts.Quiet)
|
popts := repository.PruneOptions{
|
||||||
|
DryRun: opts.DryRun,
|
||||||
|
UnsafeRecovery: opts.unsafeRecovery,
|
||||||
|
|
||||||
|
MaxUnusedBytes: opts.maxUnusedBytes,
|
||||||
|
MaxRepackBytes: opts.MaxRepackBytes,
|
||||||
|
|
||||||
|
RepackCachableOnly: opts.RepackCachableOnly,
|
||||||
|
RepackSmall: opts.RepackSmall,
|
||||||
|
RepackUncompressed: opts.RepackUncompressed,
|
||||||
|
}
|
||||||
|
|
||||||
|
plan, err := repository.PlanPrune(ctx, popts, repo, func(ctx context.Context, repo restic.Repository, usedBlobs restic.FindBlobSet) error {
|
||||||
|
return getUsedBlobs(ctx, repo, usedBlobs, ignoreSnapshots, printer)
|
||||||
|
}, printer)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if ctx.Err() != nil {
|
||||||
if opts.DryRun {
|
return ctx.Err()
|
||||||
Verbosef("\nWould have made the following changes:")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
err = printPruneStats(stats)
|
if popts.DryRun {
|
||||||
|
printer.P("\nWould have made the following changes:")
|
||||||
|
}
|
||||||
|
|
||||||
|
err = printPruneStats(printer, plan.Stats())
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -210,607 +210,55 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption
|
||||||
// Trigger GC to reset garbage collection threshold
|
// Trigger GC to reset garbage collection threshold
|
||||||
runtime.GC()
|
runtime.GC()
|
||||||
|
|
||||||
return doPrune(ctx, opts, gopts, repo, plan)
|
return plan.Execute(ctx, printer)
|
||||||
}
|
|
||||||
|
|
||||||
type pruneStats struct {
|
|
||||||
blobs struct {
|
|
||||||
used uint
|
|
||||||
duplicate uint
|
|
||||||
unused uint
|
|
||||||
remove uint
|
|
||||||
repack uint
|
|
||||||
repackrm uint
|
|
||||||
}
|
|
||||||
size struct {
|
|
||||||
used uint64
|
|
||||||
duplicate uint64
|
|
||||||
unused uint64
|
|
||||||
remove uint64
|
|
||||||
repack uint64
|
|
||||||
repackrm uint64
|
|
||||||
unref uint64
|
|
||||||
uncompressed uint64
|
|
||||||
}
|
|
||||||
packs struct {
|
|
||||||
used uint
|
|
||||||
unused uint
|
|
||||||
partlyUsed uint
|
|
||||||
unref uint
|
|
||||||
keep uint
|
|
||||||
repack uint
|
|
||||||
remove uint
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type prunePlan struct {
|
|
||||||
removePacksFirst restic.IDSet // packs to remove first (unreferenced packs)
|
|
||||||
repackPacks restic.IDSet // packs to repack
|
|
||||||
keepBlobs restic.CountedBlobSet // blobs to keep during repacking
|
|
||||||
removePacks restic.IDSet // packs to remove
|
|
||||||
ignorePacks restic.IDSet // packs to ignore when rebuilding the index
|
|
||||||
}
|
|
||||||
|
|
||||||
type packInfo struct {
|
|
||||||
usedBlobs uint
|
|
||||||
unusedBlobs uint
|
|
||||||
usedSize uint64
|
|
||||||
unusedSize uint64
|
|
||||||
tpe restic.BlobType
|
|
||||||
uncompressed bool
|
|
||||||
}
|
|
||||||
|
|
||||||
type packInfoWithID struct {
|
|
||||||
ID restic.ID
|
|
||||||
packInfo
|
|
||||||
mustCompress bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// planPrune selects which files to rewrite and which to delete and which blobs to keep.
|
|
||||||
// Also some summary statistics are returned.
|
|
||||||
func planPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (prunePlan, pruneStats, error) {
|
|
||||||
var stats pruneStats
|
|
||||||
|
|
||||||
usedBlobs, err := getUsedBlobs(ctx, repo, ignoreSnapshots, quiet)
|
|
||||||
if err != nil {
|
|
||||||
return prunePlan{}, stats, err
|
|
||||||
}
|
|
||||||
|
|
||||||
Verbosef("searching used packs...\n")
|
|
||||||
keepBlobs, indexPack, err := packInfoFromIndex(ctx, repo.Index(), usedBlobs, &stats)
|
|
||||||
if err != nil {
|
|
||||||
return prunePlan{}, stats, err
|
|
||||||
}
|
|
||||||
|
|
||||||
Verbosef("collecting packs for deletion and repacking\n")
|
|
||||||
plan, err := decidePackAction(ctx, opts, repo, indexPack, &stats, quiet)
|
|
||||||
if err != nil {
|
|
||||||
return prunePlan{}, stats, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(plan.repackPacks) != 0 {
|
|
||||||
blobCount := keepBlobs.Len()
|
|
||||||
// when repacking, we do not want to keep blobs which are
|
|
||||||
// already contained in kept packs, so delete them from keepBlobs
|
|
||||||
repo.Index().Each(ctx, func(blob restic.PackedBlob) {
|
|
||||||
if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
keepBlobs.Delete(blob.BlobHandle)
|
|
||||||
})
|
|
||||||
|
|
||||||
if keepBlobs.Len() < blobCount/2 {
|
|
||||||
// replace with copy to shrink map to necessary size if there's a chance to benefit
|
|
||||||
keepBlobs = keepBlobs.Copy()
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
// keepBlobs is only needed if packs are repacked
|
|
||||||
keepBlobs = nil
|
|
||||||
}
|
|
||||||
plan.keepBlobs = keepBlobs
|
|
||||||
|
|
||||||
return plan, stats, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs restic.CountedBlobSet, stats *pruneStats) (restic.CountedBlobSet, map[restic.ID]packInfo, error) {
|
|
||||||
// iterate over all blobs in index to find out which blobs are duplicates
|
|
||||||
// The counter in usedBlobs describes how many instances of the blob exist in the repository index
|
|
||||||
// Thus 0 == blob is missing, 1 == blob exists once, >= 2 == duplicates exist
|
|
||||||
idx.Each(ctx, func(blob restic.PackedBlob) {
|
|
||||||
bh := blob.BlobHandle
|
|
||||||
count, ok := usedBlobs[bh]
|
|
||||||
if ok {
|
|
||||||
if count < math.MaxUint8 {
|
|
||||||
// don't overflow, but saturate count at 255
|
|
||||||
// this can lead to a non-optimal pack selection, but won't cause
|
|
||||||
// problems otherwise
|
|
||||||
count++
|
|
||||||
}
|
|
||||||
|
|
||||||
usedBlobs[bh] = count
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
// Check if all used blobs have been found in index
|
|
||||||
missingBlobs := restic.NewBlobSet()
|
|
||||||
for bh, count := range usedBlobs {
|
|
||||||
if count == 0 {
|
|
||||||
// blob does not exist in any pack files
|
|
||||||
missingBlobs.Insert(bh)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(missingBlobs) != 0 {
|
|
||||||
Warnf("%v not found in the index\n\n"+
|
|
||||||
"Integrity check failed: Data seems to be missing.\n"+
|
|
||||||
"Will not start prune to prevent (additional) data loss!\n"+
|
|
||||||
"Please report this error (along with the output of the 'prune' run) at\n"+
|
|
||||||
"https://github.com/restic/restic/issues/new/choose\n", missingBlobs)
|
|
||||||
return nil, nil, errorIndexIncomplete
|
|
||||||
}
|
|
||||||
|
|
||||||
indexPack := make(map[restic.ID]packInfo)
|
|
||||||
|
|
||||||
// save computed pack header size
|
|
||||||
for pid, hdrSize := range pack.Size(ctx, idx, true) {
|
|
||||||
// initialize tpe with NumBlobTypes to indicate it's not set
|
|
||||||
indexPack[pid] = packInfo{tpe: restic.NumBlobTypes, usedSize: uint64(hdrSize)}
|
|
||||||
}
|
|
||||||
|
|
||||||
hasDuplicates := false
|
|
||||||
// iterate over all blobs in index to generate packInfo
|
|
||||||
idx.Each(ctx, func(blob restic.PackedBlob) {
|
|
||||||
ip := indexPack[blob.PackID]
|
|
||||||
|
|
||||||
// Set blob type if not yet set
|
|
||||||
if ip.tpe == restic.NumBlobTypes {
|
|
||||||
ip.tpe = blob.Type
|
|
||||||
}
|
|
||||||
|
|
||||||
// mark mixed packs with "Invalid blob type"
|
|
||||||
if ip.tpe != blob.Type {
|
|
||||||
ip.tpe = restic.InvalidBlob
|
|
||||||
}
|
|
||||||
|
|
||||||
bh := blob.BlobHandle
|
|
||||||
size := uint64(blob.Length)
|
|
||||||
dupCount := usedBlobs[bh]
|
|
||||||
switch {
|
|
||||||
case dupCount >= 2:
|
|
||||||
hasDuplicates = true
|
|
||||||
// mark as unused for now, we will later on select one copy
|
|
||||||
ip.unusedSize += size
|
|
||||||
ip.unusedBlobs++
|
|
||||||
|
|
||||||
// count as duplicate, will later on change one copy to be counted as used
|
|
||||||
stats.size.duplicate += size
|
|
||||||
stats.blobs.duplicate++
|
|
||||||
case dupCount == 1: // used blob, not duplicate
|
|
||||||
ip.usedSize += size
|
|
||||||
ip.usedBlobs++
|
|
||||||
|
|
||||||
stats.size.used += size
|
|
||||||
stats.blobs.used++
|
|
||||||
default: // unused blob
|
|
||||||
ip.unusedSize += size
|
|
||||||
ip.unusedBlobs++
|
|
||||||
|
|
||||||
stats.size.unused += size
|
|
||||||
stats.blobs.unused++
|
|
||||||
}
|
|
||||||
if !blob.IsCompressed() {
|
|
||||||
ip.uncompressed = true
|
|
||||||
}
|
|
||||||
// update indexPack
|
|
||||||
indexPack[blob.PackID] = ip
|
|
||||||
})
|
|
||||||
|
|
||||||
// if duplicate blobs exist, those will be set to either "used" or "unused":
|
|
||||||
// - mark only one occurence of duplicate blobs as used
|
|
||||||
// - if there are already some used blobs in a pack, possibly mark duplicates in this pack as "used"
|
|
||||||
// - if there are no used blobs in a pack, possibly mark duplicates as "unused"
|
|
||||||
if hasDuplicates {
|
|
||||||
// iterate again over all blobs in index (this is pretty cheap, all in-mem)
|
|
||||||
idx.Each(ctx, func(blob restic.PackedBlob) {
|
|
||||||
bh := blob.BlobHandle
|
|
||||||
count, ok := usedBlobs[bh]
|
|
||||||
// skip non-duplicate, aka. normal blobs
|
|
||||||
// count == 0 is used to mark that this was a duplicate blob with only a single occurence remaining
|
|
||||||
if !ok || count == 1 {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
ip := indexPack[blob.PackID]
|
|
||||||
size := uint64(blob.Length)
|
|
||||||
switch {
|
|
||||||
case ip.usedBlobs > 0, count == 0:
|
|
||||||
// other used blobs in pack or "last" occurence -> transition to used
|
|
||||||
ip.usedSize += size
|
|
||||||
ip.usedBlobs++
|
|
||||||
ip.unusedSize -= size
|
|
||||||
ip.unusedBlobs--
|
|
||||||
// same for the global statistics
|
|
||||||
stats.size.used += size
|
|
||||||
stats.blobs.used++
|
|
||||||
stats.size.duplicate -= size
|
|
||||||
stats.blobs.duplicate--
|
|
||||||
// let other occurences remain marked as unused
|
|
||||||
usedBlobs[bh] = 1
|
|
||||||
default:
|
|
||||||
// remain unused and decrease counter
|
|
||||||
count--
|
|
||||||
if count == 1 {
|
|
||||||
// setting count to 1 would lead to forgetting that this blob had duplicates
|
|
||||||
// thus use the special value zero. This will select the last instance of the blob for keeping.
|
|
||||||
count = 0
|
|
||||||
}
|
|
||||||
usedBlobs[bh] = count
|
|
||||||
}
|
|
||||||
// update indexPack
|
|
||||||
indexPack[blob.PackID] = ip
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sanity check. If no duplicates exist, all blobs have value 1. After handling
|
|
||||||
// duplicates, this also applies to duplicates.
|
|
||||||
for _, count := range usedBlobs {
|
|
||||||
if count != 1 {
|
|
||||||
panic("internal error during blob selection")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return usedBlobs, indexPack, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Repository, indexPack map[restic.ID]packInfo, stats *pruneStats, quiet bool) (prunePlan, error) {
|
|
||||||
removePacksFirst := restic.NewIDSet()
|
|
||||||
removePacks := restic.NewIDSet()
|
|
||||||
repackPacks := restic.NewIDSet()
|
|
||||||
|
|
||||||
var repackCandidates []packInfoWithID
|
|
||||||
var repackSmallCandidates []packInfoWithID
|
|
||||||
repoVersion := repo.Config().Version
|
|
||||||
// only repack very small files by default
|
|
||||||
targetPackSize := repo.PackSize() / 25
|
|
||||||
if opts.RepackSmall {
|
|
||||||
// consider files with at least 80% of the target size as large enough
|
|
||||||
targetPackSize = repo.PackSize() / 5 * 4
|
|
||||||
}
|
|
||||||
|
|
||||||
// loop over all packs and decide what to do
|
|
||||||
bar := newProgressMax(!quiet, uint64(len(indexPack)), "packs processed")
|
|
||||||
err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
|
|
||||||
p, ok := indexPack[id]
|
|
||||||
if !ok {
|
|
||||||
// Pack was not referenced in index and is not used => immediately remove!
|
|
||||||
Verboseff("will remove pack %v as it is unused and not indexed\n", id.Str())
|
|
||||||
removePacksFirst.Insert(id)
|
|
||||||
stats.size.unref += uint64(packSize)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if p.unusedSize+p.usedSize != uint64(packSize) && p.usedBlobs != 0 {
|
|
||||||
// Pack size does not fit and pack is needed => error
|
|
||||||
// If the pack is not needed, this is no error, the pack can
|
|
||||||
// and will be simply removed, see below.
|
|
||||||
Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic repair index'.\n",
|
|
||||||
id.Str(), p.unusedSize+p.usedSize, packSize)
|
|
||||||
return errorSizeNotMatching
|
|
||||||
}
|
|
||||||
|
|
||||||
// statistics
|
|
||||||
switch {
|
|
||||||
case p.usedBlobs == 0:
|
|
||||||
stats.packs.unused++
|
|
||||||
case p.unusedBlobs == 0:
|
|
||||||
stats.packs.used++
|
|
||||||
default:
|
|
||||||
stats.packs.partlyUsed++
|
|
||||||
}
|
|
||||||
|
|
||||||
if p.uncompressed {
|
|
||||||
stats.size.uncompressed += p.unusedSize + p.usedSize
|
|
||||||
}
|
|
||||||
mustCompress := false
|
|
||||||
if repoVersion >= 2 {
|
|
||||||
// repo v2: always repack tree blobs if uncompressed
|
|
||||||
// compress data blobs if requested
|
|
||||||
mustCompress = (p.tpe == restic.TreeBlob || opts.RepackUncompressed) && p.uncompressed
|
|
||||||
}
|
|
||||||
|
|
||||||
// decide what to do
|
|
||||||
switch {
|
|
||||||
case p.usedBlobs == 0:
|
|
||||||
// All blobs in pack are no longer used => remove pack!
|
|
||||||
removePacks.Insert(id)
|
|
||||||
stats.blobs.remove += p.unusedBlobs
|
|
||||||
stats.size.remove += p.unusedSize
|
|
||||||
|
|
||||||
case opts.RepackCachableOnly && p.tpe == restic.DataBlob:
|
|
||||||
// if this is a data pack and --repack-cacheable-only is set => keep pack!
|
|
||||||
stats.packs.keep++
|
|
||||||
|
|
||||||
case p.unusedBlobs == 0 && p.tpe != restic.InvalidBlob && !mustCompress:
|
|
||||||
if packSize >= int64(targetPackSize) {
|
|
||||||
// All blobs in pack are used and not mixed => keep pack!
|
|
||||||
stats.packs.keep++
|
|
||||||
} else {
|
|
||||||
repackSmallCandidates = append(repackSmallCandidates, packInfoWithID{ID: id, packInfo: p, mustCompress: mustCompress})
|
|
||||||
}
|
|
||||||
|
|
||||||
default:
|
|
||||||
// all other packs are candidates for repacking
|
|
||||||
repackCandidates = append(repackCandidates, packInfoWithID{ID: id, packInfo: p, mustCompress: mustCompress})
|
|
||||||
}
|
|
||||||
|
|
||||||
delete(indexPack, id)
|
|
||||||
bar.Add(1)
|
|
||||||
return nil
|
|
||||||
})
|
|
||||||
bar.Done()
|
|
||||||
if err != nil {
|
|
||||||
return prunePlan{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// At this point indexPacks contains only missing packs!
|
|
||||||
|
|
||||||
// missing packs that are not needed can be ignored
|
|
||||||
ignorePacks := restic.NewIDSet()
|
|
||||||
for id, p := range indexPack {
|
|
||||||
if p.usedBlobs == 0 {
|
|
||||||
ignorePacks.Insert(id)
|
|
||||||
stats.blobs.remove += p.unusedBlobs
|
|
||||||
stats.size.remove += p.unusedSize
|
|
||||||
delete(indexPack, id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(indexPack) != 0 {
|
|
||||||
Warnf("The index references %d needed pack files which are missing from the repository:\n", len(indexPack))
|
|
||||||
for id := range indexPack {
|
|
||||||
Warnf(" %v\n", id)
|
|
||||||
}
|
|
||||||
return prunePlan{}, errorPacksMissing
|
|
||||||
}
|
|
||||||
if len(ignorePacks) != 0 {
|
|
||||||
Warnf("Missing but unneeded pack files are referenced in the index, will be repaired\n")
|
|
||||||
for id := range ignorePacks {
|
|
||||||
Warnf("will forget missing pack file %v\n", id)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(repackSmallCandidates) < 10 {
|
|
||||||
// too few small files to be worth the trouble, this also prevents endlessly repacking
|
|
||||||
// if there is just a single pack file below the target size
|
|
||||||
stats.packs.keep += uint(len(repackSmallCandidates))
|
|
||||||
} else {
|
|
||||||
repackCandidates = append(repackCandidates, repackSmallCandidates...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sort repackCandidates such that packs with highest ratio unused/used space are picked first.
|
|
||||||
// This is equivalent to sorting by unused / total space.
|
|
||||||
// Instead of unused[i] / used[i] > unused[j] / used[j] we use
|
|
||||||
// unused[i] * used[j] > unused[j] * used[i] as uint32*uint32 < uint64
|
|
||||||
// Moreover packs containing trees and too small packs are sorted to the beginning
|
|
||||||
sort.Slice(repackCandidates, func(i, j int) bool {
|
|
||||||
pi := repackCandidates[i].packInfo
|
|
||||||
pj := repackCandidates[j].packInfo
|
|
||||||
switch {
|
|
||||||
case pi.tpe != restic.DataBlob && pj.tpe == restic.DataBlob:
|
|
||||||
return true
|
|
||||||
case pj.tpe != restic.DataBlob && pi.tpe == restic.DataBlob:
|
|
||||||
return false
|
|
||||||
case pi.unusedSize+pi.usedSize < uint64(targetPackSize) && pj.unusedSize+pj.usedSize >= uint64(targetPackSize):
|
|
||||||
return true
|
|
||||||
case pj.unusedSize+pj.usedSize < uint64(targetPackSize) && pi.unusedSize+pi.usedSize >= uint64(targetPackSize):
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return pi.unusedSize*pj.usedSize > pj.unusedSize*pi.usedSize
|
|
||||||
})
|
|
||||||
|
|
||||||
repack := func(id restic.ID, p packInfo) {
|
|
||||||
repackPacks.Insert(id)
|
|
||||||
stats.blobs.repack += p.unusedBlobs + p.usedBlobs
|
|
||||||
stats.size.repack += p.unusedSize + p.usedSize
|
|
||||||
stats.blobs.repackrm += p.unusedBlobs
|
|
||||||
stats.size.repackrm += p.unusedSize
|
|
||||||
if p.uncompressed {
|
|
||||||
stats.size.uncompressed -= p.unusedSize + p.usedSize
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// calculate limit for number of unused bytes in the repo after repacking
|
|
||||||
maxUnusedSizeAfter := opts.maxUnusedBytes(stats.size.used)
|
|
||||||
|
|
||||||
for _, p := range repackCandidates {
|
|
||||||
reachedUnusedSizeAfter := (stats.size.unused-stats.size.remove-stats.size.repackrm < maxUnusedSizeAfter)
|
|
||||||
reachedRepackSize := stats.size.repack+p.unusedSize+p.usedSize >= opts.MaxRepackBytes
|
|
||||||
packIsLargeEnough := p.unusedSize+p.usedSize >= uint64(targetPackSize)
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case reachedRepackSize:
|
|
||||||
stats.packs.keep++
|
|
||||||
|
|
||||||
case p.tpe != restic.DataBlob, p.mustCompress:
|
|
||||||
// repacking non-data packs / uncompressed-trees is only limited by repackSize
|
|
||||||
repack(p.ID, p.packInfo)
|
|
||||||
|
|
||||||
case reachedUnusedSizeAfter && packIsLargeEnough:
|
|
||||||
// for all other packs stop repacking if tolerated unused size is reached.
|
|
||||||
stats.packs.keep++
|
|
||||||
|
|
||||||
default:
|
|
||||||
repack(p.ID, p.packInfo)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
stats.packs.unref = uint(len(removePacksFirst))
|
|
||||||
stats.packs.repack = uint(len(repackPacks))
|
|
||||||
stats.packs.remove = uint(len(removePacks))
|
|
||||||
|
|
||||||
if repo.Config().Version < 2 {
|
|
||||||
// compression not supported for repository format version 1
|
|
||||||
stats.size.uncompressed = 0
|
|
||||||
}
|
|
||||||
|
|
||||||
return prunePlan{removePacksFirst: removePacksFirst,
|
|
||||||
removePacks: removePacks,
|
|
||||||
repackPacks: repackPacks,
|
|
||||||
ignorePacks: ignorePacks,
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// printPruneStats prints out the statistics
|
// printPruneStats prints out the statistics
|
||||||
func printPruneStats(stats pruneStats) error {
|
func printPruneStats(printer progress.Printer, stats repository.PruneStats) error {
|
||||||
Verboseff("\nused: %10d blobs / %s\n", stats.blobs.used, ui.FormatBytes(stats.size.used))
|
printer.V("\nused: %10d blobs / %s\n", stats.Blobs.Used, ui.FormatBytes(stats.Size.Used))
|
||||||
if stats.blobs.duplicate > 0 {
|
if stats.Blobs.Duplicate > 0 {
|
||||||
Verboseff("duplicates: %10d blobs / %s\n", stats.blobs.duplicate, ui.FormatBytes(stats.size.duplicate))
|
printer.V("duplicates: %10d blobs / %s\n", stats.Blobs.Duplicate, ui.FormatBytes(stats.Size.Duplicate))
|
||||||
}
|
}
|
||||||
Verboseff("unused: %10d blobs / %s\n", stats.blobs.unused, ui.FormatBytes(stats.size.unused))
|
printer.V("unused: %10d blobs / %s\n", stats.Blobs.Unused, ui.FormatBytes(stats.Size.Unused))
|
||||||
if stats.size.unref > 0 {
|
if stats.Size.Unref > 0 {
|
||||||
Verboseff("unreferenced: %s\n", ui.FormatBytes(stats.size.unref))
|
printer.V("unreferenced: %s\n", ui.FormatBytes(stats.Size.Unref))
|
||||||
}
|
}
|
||||||
totalBlobs := stats.blobs.used + stats.blobs.unused + stats.blobs.duplicate
|
totalBlobs := stats.Blobs.Used + stats.Blobs.Unused + stats.Blobs.Duplicate
|
||||||
totalSize := stats.size.used + stats.size.duplicate + stats.size.unused + stats.size.unref
|
totalSize := stats.Size.Used + stats.Size.Duplicate + stats.Size.Unused + stats.Size.Unref
|
||||||
unusedSize := stats.size.duplicate + stats.size.unused
|
unusedSize := stats.Size.Duplicate + stats.Size.Unused
|
||||||
Verboseff("total: %10d blobs / %s\n", totalBlobs, ui.FormatBytes(totalSize))
|
printer.V("total: %10d blobs / %s\n", totalBlobs, ui.FormatBytes(totalSize))
|
||||||
Verboseff("unused size: %s of total size\n", ui.FormatPercent(unusedSize, totalSize))
|
printer.V("unused size: %s of total size\n", ui.FormatPercent(unusedSize, totalSize))
|
||||||
|
|
||||||
Verbosef("\nto repack: %10d blobs / %s\n", stats.blobs.repack, ui.FormatBytes(stats.size.repack))
|
printer.P("\nto repack: %10d blobs / %s\n", stats.Blobs.Repack, ui.FormatBytes(stats.Size.Repack))
|
||||||
Verbosef("this removes: %10d blobs / %s\n", stats.blobs.repackrm, ui.FormatBytes(stats.size.repackrm))
|
printer.P("this removes: %10d blobs / %s\n", stats.Blobs.Repackrm, ui.FormatBytes(stats.Size.Repackrm))
|
||||||
Verbosef("to delete: %10d blobs / %s\n", stats.blobs.remove, ui.FormatBytes(stats.size.remove+stats.size.unref))
|
printer.P("to delete: %10d blobs / %s\n", stats.Blobs.Remove, ui.FormatBytes(stats.Size.Remove+stats.Size.Unref))
|
||||||
totalPruneSize := stats.size.remove + stats.size.repackrm + stats.size.unref
|
totalPruneSize := stats.Size.Remove + stats.Size.Repackrm + stats.Size.Unref
|
||||||
Verbosef("total prune: %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, ui.FormatBytes(totalPruneSize))
|
printer.P("total prune: %10d blobs / %s\n", stats.Blobs.Remove+stats.Blobs.Repackrm, ui.FormatBytes(totalPruneSize))
|
||||||
if stats.size.uncompressed > 0 {
|
if stats.Size.Uncompressed > 0 {
|
||||||
Verbosef("not yet compressed: %s\n", ui.FormatBytes(stats.size.uncompressed))
|
printer.P("not yet compressed: %s\n", ui.FormatBytes(stats.Size.Uncompressed))
|
||||||
}
|
}
|
||||||
Verbosef("remaining: %10d blobs / %s\n", totalBlobs-(stats.blobs.remove+stats.blobs.repackrm), ui.FormatBytes(totalSize-totalPruneSize))
|
printer.P("remaining: %10d blobs / %s\n", totalBlobs-(stats.Blobs.Remove+stats.Blobs.Repackrm), ui.FormatBytes(totalSize-totalPruneSize))
|
||||||
unusedAfter := unusedSize - stats.size.remove - stats.size.repackrm
|
unusedAfter := unusedSize - stats.Size.Remove - stats.Size.Repackrm
|
||||||
Verbosef("unused size after prune: %s (%s of remaining size)\n",
|
printer.P("unused size after prune: %s (%s of remaining size)\n",
|
||||||
ui.FormatBytes(unusedAfter), ui.FormatPercent(unusedAfter, totalSize-totalPruneSize))
|
ui.FormatBytes(unusedAfter), ui.FormatPercent(unusedAfter, totalSize-totalPruneSize))
|
||||||
Verbosef("\n")
|
printer.P("\n")
|
||||||
Verboseff("totally used packs: %10d\n", stats.packs.used)
|
printer.V("totally used packs: %10d\n", stats.Packs.Used)
|
||||||
Verboseff("partly used packs: %10d\n", stats.packs.partlyUsed)
|
printer.V("partly used packs: %10d\n", stats.Packs.PartlyUsed)
|
||||||
Verboseff("unused packs: %10d\n\n", stats.packs.unused)
|
printer.V("unused packs: %10d\n\n", stats.Packs.Unused)
|
||||||
|
|
||||||
Verboseff("to keep: %10d packs\n", stats.packs.keep)
|
printer.V("to keep: %10d packs\n", stats.Packs.Keep)
|
||||||
Verboseff("to repack: %10d packs\n", stats.packs.repack)
|
printer.V("to repack: %10d packs\n", stats.Packs.Repack)
|
||||||
Verboseff("to delete: %10d packs\n", stats.packs.remove)
|
printer.V("to delete: %10d packs\n", stats.Packs.Remove)
|
||||||
if stats.packs.unref > 0 {
|
if stats.Packs.Unref > 0 {
|
||||||
Verboseff("to delete: %10d unreferenced packs\n\n", stats.packs.unref)
|
printer.V("to delete: %10d unreferenced packs\n\n", stats.Packs.Unref)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// doPrune does the actual pruning:
|
func getUsedBlobs(ctx context.Context, repo restic.Repository, usedBlobs restic.FindBlobSet, ignoreSnapshots restic.IDSet, printer progress.Printer) error {
|
||||||
// - remove unreferenced packs first
|
|
||||||
// - repack given pack files while keeping the given blobs
|
|
||||||
// - rebuild the index while ignoring all files that will be deleted
|
|
||||||
// - delete the files
|
|
||||||
// plan.removePacks and plan.ignorePacks are modified in this function.
|
|
||||||
func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo restic.Repository, plan prunePlan) (err error) {
|
|
||||||
if opts.DryRun {
|
|
||||||
if !gopts.JSON && gopts.verbosity >= 2 {
|
|
||||||
Printf("Repeated prune dry-runs can report slightly different amounts of data to keep or repack. This is expected behavior.\n\n")
|
|
||||||
if len(plan.removePacksFirst) > 0 {
|
|
||||||
Printf("Would have removed the following unreferenced packs:\n%v\n\n", plan.removePacksFirst)
|
|
||||||
}
|
|
||||||
Printf("Would have repacked and removed the following packs:\n%v\n\n", plan.repackPacks)
|
|
||||||
Printf("Would have removed the following no longer used packs:\n%v\n\n", plan.removePacks)
|
|
||||||
}
|
|
||||||
// Always quit here if DryRun was set!
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// unreferenced packs can be safely deleted first
|
|
||||||
if len(plan.removePacksFirst) != 0 {
|
|
||||||
Verbosef("deleting unreferenced packs\n")
|
|
||||||
DeleteFiles(ctx, gopts, repo, plan.removePacksFirst, restic.PackFile)
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(plan.repackPacks) != 0 {
|
|
||||||
Verbosef("repacking packs\n")
|
|
||||||
bar := newProgressMax(!gopts.Quiet, uint64(len(plan.repackPacks)), "packs repacked")
|
|
||||||
_, err := repository.Repack(ctx, repo, repo, plan.repackPacks, plan.keepBlobs, bar)
|
|
||||||
bar.Done()
|
|
||||||
if err != nil {
|
|
||||||
return errors.Fatal(err.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
// Also remove repacked packs
|
|
||||||
plan.removePacks.Merge(plan.repackPacks)
|
|
||||||
|
|
||||||
if len(plan.keepBlobs) != 0 {
|
|
||||||
Warnf("%v was not repacked\n\n"+
|
|
||||||
"Integrity check failed.\n"+
|
|
||||||
"Please report this error (along with the output of the 'prune' run) at\n"+
|
|
||||||
"https://github.com/restic/restic/issues/new/choose\n", plan.keepBlobs)
|
|
||||||
return errors.Fatal("internal error: blobs were not repacked")
|
|
||||||
}
|
|
||||||
|
|
||||||
// allow GC of the blob set
|
|
||||||
plan.keepBlobs = nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(plan.ignorePacks) == 0 {
|
|
||||||
plan.ignorePacks = plan.removePacks
|
|
||||||
} else {
|
|
||||||
plan.ignorePacks.Merge(plan.removePacks)
|
|
||||||
}
|
|
||||||
|
|
||||||
if opts.unsafeRecovery {
|
|
||||||
Verbosef("deleting index files\n")
|
|
||||||
indexFiles := repo.Index().(*index.MasterIndex).IDs()
|
|
||||||
err = DeleteFilesChecked(ctx, gopts, repo, indexFiles, restic.IndexFile)
|
|
||||||
if err != nil {
|
|
||||||
return errors.Fatalf("%s", err)
|
|
||||||
}
|
|
||||||
} else if len(plan.ignorePacks) != 0 {
|
|
||||||
err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil)
|
|
||||||
if err != nil {
|
|
||||||
return errors.Fatalf("%s", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(plan.removePacks) != 0 {
|
|
||||||
Verbosef("removing %d old packs\n", len(plan.removePacks))
|
|
||||||
DeleteFiles(ctx, gopts, repo, plan.removePacks, restic.PackFile)
|
|
||||||
}
|
|
||||||
|
|
||||||
if opts.unsafeRecovery {
|
|
||||||
_, err = writeIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil)
|
|
||||||
if err != nil {
|
|
||||||
return errors.Fatalf("%s", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Verbosef("done\n")
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func writeIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) (restic.IDSet, error) {
|
|
||||||
Verbosef("rebuilding index\n")
|
|
||||||
|
|
||||||
bar := newProgressMax(!gopts.Quiet, 0, "packs processed")
|
|
||||||
obsoleteIndexes, err := repo.Index().Save(ctx, repo, removePacks, extraObsolete, bar)
|
|
||||||
bar.Done()
|
|
||||||
return obsoleteIndexes, err
|
|
||||||
}
|
|
||||||
|
|
||||||
func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs) error {
|
|
||||||
obsoleteIndexes, err := writeIndexFiles(ctx, gopts, repo, removePacks, extraObsolete)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
Verbosef("deleting obsolete index files\n")
|
|
||||||
return DeleteFilesChecked(ctx, gopts, repo, obsoleteIndexes, restic.IndexFile)
|
|
||||||
}
|
|
||||||
|
|
||||||
func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (usedBlobs restic.CountedBlobSet, err error) {
|
|
||||||
var snapshotTrees restic.IDs
|
var snapshotTrees restic.IDs
|
||||||
Verbosef("loading all snapshots...\n")
|
printer.P("loading all snapshots...\n")
|
||||||
err = restic.ForAllSnapshots(ctx, repo.Backend(), repo, ignoreSnapshots,
|
err := restic.ForAllSnapshots(ctx, repo, repo, ignoreSnapshots,
|
||||||
func(id restic.ID, sn *restic.Snapshot, err error) error {
|
func(id restic.ID, sn *restic.Snapshot, err error) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
debug.Log("failed to load snapshot %v (error %v)", id, err)
|
debug.Log("failed to load snapshot %v (error %v)", id, err)
|
||||||
|
@ -821,23 +269,14 @@ func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots r
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.Fatalf("failed loading snapshot: %v", err)
|
return errors.Fatalf("failed loading snapshot: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
Verbosef("finding data that is still in use for %d snapshots\n", len(snapshotTrees))
|
printer.P("finding data that is still in use for %d snapshots\n", len(snapshotTrees))
|
||||||
|
|
||||||
usedBlobs = restic.NewCountedBlobSet()
|
bar := printer.NewCounter("snapshots")
|
||||||
|
bar.SetMax(uint64(len(snapshotTrees)))
|
||||||
bar := newProgressMax(!quiet, uint64(len(snapshotTrees)), "snapshots")
|
|
||||||
defer bar.Done()
|
defer bar.Done()
|
||||||
|
|
||||||
err = restic.FindUsedBlobs(ctx, repo, snapshotTrees, usedBlobs, bar)
|
return restic.FindUsedBlobs(ctx, repo, snapshotTrees, usedBlobs, bar)
|
||||||
if err != nil {
|
|
||||||
if repo.Backend().IsNotExist(err) {
|
|
||||||
return nil, errors.Fatal("unable to load a tree from the repository: " + err.Error())
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return usedBlobs, nil
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -6,17 +6,21 @@ import (
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/restic/restic/internal/restic"
|
"github.com/restic/restic/internal/backend"
|
||||||
|
"github.com/restic/restic/internal/repository"
|
||||||
rtest "github.com/restic/restic/internal/test"
|
rtest "github.com/restic/restic/internal/test"
|
||||||
|
"github.com/restic/restic/internal/ui/termstatus"
|
||||||
)
|
)
|
||||||
|
|
||||||
func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
|
func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
|
||||||
oldHook := gopts.backendTestHook
|
oldHook := gopts.backendTestHook
|
||||||
gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil }
|
gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil }
|
||||||
defer func() {
|
defer func() {
|
||||||
gopts.backendTestHook = oldHook
|
gopts.backendTestHook = oldHook
|
||||||
}()
|
}()
|
||||||
rtest.OK(t, runPrune(context.TODO(), opts, gopts))
|
rtest.OK(t, withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
|
||||||
|
return runPrune(context.TODO(), opts, gopts, term)
|
||||||
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPrune(t *testing.T) {
|
func TestPrune(t *testing.T) {
|
||||||
|
@ -31,7 +35,7 @@ func testPruneVariants(t *testing.T, unsafeNoSpaceRecovery bool) {
|
||||||
}
|
}
|
||||||
t.Run("0"+suffix, func(t *testing.T) {
|
t.Run("0"+suffix, func(t *testing.T) {
|
||||||
opts := PruneOptions{MaxUnused: "0%", unsafeRecovery: unsafeNoSpaceRecovery}
|
opts := PruneOptions{MaxUnused: "0%", unsafeRecovery: unsafeNoSpaceRecovery}
|
||||||
checkOpts := CheckOptions{ReadData: true, CheckUnused: true}
|
checkOpts := CheckOptions{ReadData: true, CheckUnused: !unsafeNoSpaceRecovery}
|
||||||
testPrune(t, opts, checkOpts)
|
testPrune(t, opts, checkOpts)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -71,7 +75,7 @@ func createPrunableRepo(t *testing.T, env *testEnvironment) {
|
||||||
testListSnapshots(t, env.gopts, 3)
|
testListSnapshots(t, env.gopts, 3)
|
||||||
|
|
||||||
testRunForgetJSON(t, env.gopts)
|
testRunForgetJSON(t, env.gopts)
|
||||||
testRunForget(t, env.gopts, firstSnapshot.String())
|
testRunForget(t, env.gopts, ForgetOptions{}, firstSnapshot.String())
|
||||||
}
|
}
|
||||||
|
|
||||||
func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) {
|
func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) {
|
||||||
|
@ -81,7 +85,12 @@ func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) {
|
||||||
DryRun: true,
|
DryRun: true,
|
||||||
Last: 1,
|
Last: 1,
|
||||||
}
|
}
|
||||||
return runForget(context.TODO(), opts, gopts, args)
|
pruneOpts := PruneOptions{
|
||||||
|
MaxUnused: "5%",
|
||||||
|
}
|
||||||
|
return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
|
||||||
|
return runForget(context.TODO(), opts, pruneOpts, gopts, term, args)
|
||||||
|
})
|
||||||
})
|
})
|
||||||
rtest.OK(t, err)
|
rtest.OK(t, err)
|
||||||
|
|
||||||
|
@ -102,7 +111,9 @@ func testPrune(t *testing.T, pruneOpts PruneOptions, checkOpts CheckOptions) {
|
||||||
|
|
||||||
createPrunableRepo(t, env)
|
createPrunableRepo(t, env)
|
||||||
testRunPrune(t, env.gopts, pruneOpts)
|
testRunPrune(t, env.gopts, pruneOpts)
|
||||||
rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil))
|
rtest.OK(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
|
||||||
|
return runCheck(context.TODO(), checkOpts, env.gopts, nil, term)
|
||||||
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
var pruneDefaultOptions = PruneOptions{MaxUnused: "5%"}
|
var pruneDefaultOptions = PruneOptions{MaxUnused: "5%"}
|
||||||
|
@ -120,7 +131,7 @@ func TestPruneWithDamagedRepository(t *testing.T) {
|
||||||
// create and delete snapshot to create unused blobs
|
// create and delete snapshot to create unused blobs
|
||||||
testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "2")}, opts, env.gopts)
|
testRunBackup(t, "", []string{filepath.Join(env.testdata, "0", "0", "9", "2")}, opts, env.gopts)
|
||||||
firstSnapshot := testListSnapshots(t, env.gopts, 1)[0]
|
firstSnapshot := testListSnapshots(t, env.gopts, 1)[0]
|
||||||
testRunForget(t, env.gopts, firstSnapshot.String())
|
testRunForget(t, env.gopts, ForgetOptions{}, firstSnapshot.String())
|
||||||
|
|
||||||
oldPacks := listPacks(env.gopts, t)
|
oldPacks := listPacks(env.gopts, t)
|
||||||
|
|
||||||
|
@ -130,12 +141,14 @@ func TestPruneWithDamagedRepository(t *testing.T) {
|
||||||
removePacksExcept(env.gopts, t, oldPacks, false)
|
removePacksExcept(env.gopts, t, oldPacks, false)
|
||||||
|
|
||||||
oldHook := env.gopts.backendTestHook
|
oldHook := env.gopts.backendTestHook
|
||||||
env.gopts.backendTestHook = func(r restic.Backend) (restic.Backend, error) { return newListOnceBackend(r), nil }
|
env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) { return newListOnceBackend(r), nil }
|
||||||
defer func() {
|
defer func() {
|
||||||
env.gopts.backendTestHook = oldHook
|
env.gopts.backendTestHook = oldHook
|
||||||
}()
|
}()
|
||||||
// prune should fail
|
// prune should fail
|
||||||
rtest.Assert(t, runPrune(context.TODO(), pruneDefaultOptions, env.gopts) == errorPacksMissing,
|
rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
|
||||||
|
return runPrune(context.TODO(), pruneDefaultOptions, env.gopts, term)
|
||||||
|
}) == repository.ErrPacksMissing,
|
||||||
"prune should have reported index not complete error")
|
"prune should have reported index not complete error")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -207,7 +220,9 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
|
||||||
if checkOK {
|
if checkOK {
|
||||||
testRunCheck(t, env.gopts)
|
testRunCheck(t, env.gopts)
|
||||||
} else {
|
} else {
|
||||||
rtest.Assert(t, runCheck(context.TODO(), optionsCheck, env.gopts, nil) != nil,
|
rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
|
||||||
|
return runCheck(context.TODO(), optionsCheck, env.gopts, nil, term)
|
||||||
|
}) != nil,
|
||||||
"check should have reported an error")
|
"check should have reported an error")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -215,7 +230,9 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
|
||||||
testRunPrune(t, env.gopts, optionsPrune)
|
testRunPrune(t, env.gopts, optionsPrune)
|
||||||
testRunCheck(t, env.gopts)
|
testRunCheck(t, env.gopts)
|
||||||
} else {
|
} else {
|
||||||
rtest.Assert(t, runPrune(context.TODO(), optionsPrune, env.gopts) != nil,
|
rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
|
||||||
|
return runPrune(context.TODO(), optionsPrune, env.gopts, term)
|
||||||
|
}) != nil,
|
||||||
"prune should have reported an error")
|
"prune should have reported an error")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue