mirror of https://github.com/restic/restic.git
Compare commits
84 Commits
accf40e3db
...
bc8562eedf
Author | SHA1 | Date |
---|---|---|
Aneesh N | bc8562eedf | |
Michael Eischer | faffd15d13 | |
Michael Eischer | 347e9d0765 | |
Altan Orhon | 871ea1eaf3 | |
Michael Eischer | a7b5e09902 | |
Michael Eischer | 3f9d50865d | |
Michael Eischer | 5f263752d7 | |
Michael Eischer | 484dbb1cf4 | |
Michael Eischer | 940a3159b5 | |
Michael Eischer | 31624aeffd | |
Michael Eischer | 910927670f | |
Michael Eischer | 6f2a4dea21 | |
Michael Eischer | 699ef5e9de | |
Michael Eischer | eb710a28e8 | |
Michael Eischer | 86c7909f41 | |
Michael Eischer | 93135dc705 | |
Michael Eischer | 21a7cb405c | |
Michael Eischer | b15d867414 | |
Michael Eischer | 2e6c43c695 | |
Michael Eischer | f7632de3d6 | |
Michael Eischer | 6c6dceade3 | |
Michael Eischer | 10355c3fb6 | |
Michael Eischer | 228b35f074 | |
will-ca | 6aced61c72 | |
Michael Eischer | 4d22412e0c | |
coderwander | a82ed71de7 | |
Michael Eischer | 2173c69280 | |
Michael Eischer | 001bb71676 | |
Michael Eischer | c9191ea72c | |
Michael Eischer | 09587e6c08 | |
Michael Eischer | defd7ae729 | |
Michael Eischer | 038586dc9d | |
Michael Eischer | d8622c86eb | |
Michael Eischer | 8d507c1372 | |
Michael Eischer | 310db03c0e | |
Michael Eischer | 7d1b9cde34 | |
Michael Eischer | b25fc2c89d | |
Michael Eischer | c65459cd8a | |
Michael Eischer | eda9f7beb4 | |
Michael Eischer | 35277b7797 | |
Michael Eischer | 7ba5e95a82 | |
Michael Eischer | 4c9a10ca37 | |
Michael Eischer | 85e4021619 | |
Michael Eischer | fc3b548625 | |
Michael Eischer | df9d4b455d | |
Michael Eischer | 866ddf5698 | |
Michael Eischer | 32a234b67e | |
Michael Eischer | 739d11c2eb | |
Michael Eischer | 0747cf5319 | |
Aneesh Nireshwalia | ada8d09441 | |
Aneesh N | cd67905d60 | |
Aneesh Nireshwalia | ff75c9de4a | |
Aneesh N | d556942afb | |
Aneesh N | b0199616d8 | |
Aneesh N | fa9aa3557b | |
Aneesh Nireshwalia | 2ff957cc05 | |
Aneesh N | 83c1740425 | |
Aneesh Nireshwalia | e1852bc727 | |
Aneesh Nireshwalia | 02381b8eeb | |
Aneesh Nireshwalia | 53a62d452c | |
Aneesh Nireshwalia | bc3331082c | |
Aneesh Nireshwalia | 4275870876 | |
Aneesh Nireshwalia | aad1cafe97 | |
Aneesh Nireshwalia | 8a74de8b67 | |
Aneesh Nireshwalia | 4b02684f69 | |
Aneesh Nireshwalia | db9d46c5e5 | |
Aneesh Nireshwalia | 8f49cdbb86 | |
Aneesh Nireshwalia | 4b0f27a0b8 | |
Aneesh Nireshwalia | 4186709165 | |
Aneesh Nireshwalia | 2882872736 | |
Aneesh Nireshwalia | c9dba2cdd7 | |
Aneesh Nireshwalia | a3e9be1656 | |
Aneesh Nireshwalia | d062a82896 | |
Aneesh Nireshwalia | 46c26643a7 | |
Aneesh Nireshwalia | aab8a5f36f | |
Aneesh Nireshwalia | 59ecedacaf | |
Aneesh Nireshwalia | ffea194387 | |
Aneesh Nireshwalia | 3f1673d6aa | |
Aneesh Nireshwalia | 11d1b95f4d | |
Aneesh Nireshwalia | 5ff9f58fbb | |
Aneesh Nireshwalia | bc0327c884 | |
Aneesh Nireshwalia | 46c3dc618a | |
Aneesh Nireshwalia | 8a7006d12c | |
Aneesh N | 375a9b7940 |
|
@ -25,23 +25,27 @@ jobs:
|
|||
- job_name: Windows
|
||||
go: 1.22.x
|
||||
os: windows-latest
|
||||
test_smb: true
|
||||
|
||||
- job_name: macOS
|
||||
go: 1.22.x
|
||||
os: macOS-latest
|
||||
test_fuse: false
|
||||
test_smb: true
|
||||
|
||||
- job_name: Linux
|
||||
go: 1.22.x
|
||||
os: ubuntu-latest
|
||||
test_cloud_backends: true
|
||||
test_fuse: true
|
||||
test_smb: true
|
||||
check_changelog: true
|
||||
|
||||
- job_name: Linux (race)
|
||||
go: 1.22.x
|
||||
os: ubuntu-latest
|
||||
test_fuse: true
|
||||
test_smb: true
|
||||
test_opts: "-race"
|
||||
|
||||
- job_name: Linux
|
||||
|
@ -53,11 +57,13 @@ jobs:
|
|||
go: 1.20.x
|
||||
os: ubuntu-latest
|
||||
test_fuse: true
|
||||
test_smb: true
|
||||
|
||||
- job_name: Linux
|
||||
go: 1.19.x
|
||||
os: ubuntu-latest
|
||||
test_fuse: true
|
||||
test_smb: true
|
||||
|
||||
name: ${{ matrix.job_name }} Go ${{ matrix.go }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
@ -96,6 +102,97 @@ jobs:
|
|||
chmod 755 $HOME/bin/rclone
|
||||
rm -rf rclone*
|
||||
|
||||
smbuser="smbuser"
|
||||
smbpass="mGoWwqvgdnwtmh07"
|
||||
|
||||
if [ "$RUNNER_OS" == "macOS" ]; then
|
||||
smbhome=/Users/$smbuser
|
||||
echo "Get computer name"
|
||||
computername=$(sudo -S scutil --get ComputerName)
|
||||
echo "Create smb user"
|
||||
sudo dscl . -create $smbhome
|
||||
sudo dscl . -create $smbhome UserShell /bin/bash
|
||||
sudo dscl . -create $smbhome RealName $smbuser
|
||||
LastID=`dscl . -list /Users UniqueID | awk '{print $2}' | sort -n | tail -1`
|
||||
NextID=$((LastID + 1))
|
||||
sudo dscl . -create $smbhome UniqueID $NextID
|
||||
sudo dscl . -create $smbhome PrimaryGroupID 80
|
||||
sudo dscl . -create $smbhome NFSHomeDirectory $smbhome
|
||||
sudo dscl . -passwd $smbhome $smbpass
|
||||
sudo dscl . -append /Groups/admin GroupMembership $smbuser
|
||||
echo "Make home dir"
|
||||
cd /Users/
|
||||
sudo createhomedir -u $smbuser -c
|
||||
sudo mkdir $smbhome/smbshare
|
||||
sudo chown -R $smbuser $smbhome
|
||||
sudo chmod -R 755 $smbhome
|
||||
echo "Setup smb share"
|
||||
sudo sharing -a $smbhome/smbshare -S smbuser -n smbuser -s 001
|
||||
echo "Enable share for os user"
|
||||
sudo pwpolicy -u $smbuser -sethashtypes SMB-NT off
|
||||
sudo pwpolicy -u $smbuser -enableuser
|
||||
echo "Export domain"
|
||||
if [[ $computername != *.local ]]; then computername=$computername".local"; fi
|
||||
export RESTIC_SMB_DOMAIN=$computername
|
||||
sudo pwpolicy -u $smbuser -sethashtypes SMB-NT on
|
||||
sudo dscl . -passwd /Users/$smbuser $smbpass
|
||||
sudo launchctl load -w /System/Library/LaunchDaemons/com.apple.smbd.plist
|
||||
sudo defaults write /Library/Preferences/SystemConfiguration/com.apple.smb.server.plist EnabledServices -array disk
|
||||
|
||||
else
|
||||
echo "install samba"
|
||||
sudo apt-get update
|
||||
sudo apt-get install samba -y
|
||||
|
||||
echo "Allow Samba in firewall"
|
||||
sudo ufw allow 'Samba'
|
||||
|
||||
echo "modifying samba config"
|
||||
echo '' | sudo tee -a /etc/samba/smb.conf
|
||||
echo ' interfaces = 127.0.0.0/8 eth0' | sudo tee -a /etc/samba/smb.conf
|
||||
echo ' bind interfaces only = yes' | sudo tee -a /etc/samba/smb.conf
|
||||
echo '' | sudo tee -a /etc/samba/smb.conf
|
||||
echo "[$smbuser]" | sudo tee -a /etc/samba/smb.conf
|
||||
echo ' comment = Samba on Ubuntu' | sudo tee -a /etc/samba/smb.conf
|
||||
echo " path = /samba/$smbuser" | sudo tee -a /etc/samba/smb.conf
|
||||
echo ' browseable = yes' | sudo tee -a /etc/samba/smb.conf
|
||||
echo ' read only = no' | sudo tee -a /etc/samba/smb.conf
|
||||
echo ' force create mode = 0660' | sudo tee -a /etc/samba/smb.conf
|
||||
echo ' force directory mode = 2770' | sudo tee -a /etc/samba/smb.conf
|
||||
echo " valid users = $smbuser" | sudo tee -a /etc/samba/smb.conf
|
||||
|
||||
echo "restart services"
|
||||
sudo systemctl restart smbd
|
||||
sudo systemctl restart nmbd
|
||||
|
||||
echo "create samba share directory"
|
||||
sudo mkdir /samba
|
||||
|
||||
echo "change sambashare group"
|
||||
sudo chgrp sambashare /samba
|
||||
|
||||
echo "add samba user"
|
||||
sudo id -u "$smbuser" &>/dev/null || sudo useradd -M -d "/samba/$smbuser" -s /usr/sbin/nologin -G sambashare "$smbuser"
|
||||
|
||||
echo "create samba share user directory"
|
||||
sudo mkdir "/samba/$smbuser"
|
||||
|
||||
echo "change samba share user directory ownership"
|
||||
sudo chown "$smbuser":sambashare "/samba/$smbuser"
|
||||
|
||||
echo "modify permissions on samba share user directory"
|
||||
sudo chmod 2770 "/samba/$smbuser"
|
||||
|
||||
echo "change smb password"
|
||||
(echo "$smbpass"; echo "$smbpass") | sudo smbpasswd -a "$smbuser"
|
||||
|
||||
echo "enable samba user"
|
||||
sudo smbpasswd -e "$smbuser"
|
||||
|
||||
echo "restart services"
|
||||
sudo systemctl restart smbd
|
||||
sudo systemctl restart nmbd
|
||||
fi
|
||||
# add $HOME/bin to path ($GOBIN was already added to the path by setup-go@v3)
|
||||
echo $HOME/bin >> $GITHUB_PATH
|
||||
if: matrix.os == 'ubuntu-latest' || matrix.os == 'macOS-latest'
|
||||
|
@ -135,6 +232,26 @@ jobs:
|
|||
Invoke-WebRequest https://github.com/restic/test-assets/raw/master/libiconv-1.8-1-bin.zip -OutFile libiconv.zip
|
||||
unzip libiconv.zip
|
||||
|
||||
# Create new smbshare
|
||||
$smbuser="smbuser"
|
||||
$smbpass="mGoWwqvgdnwtmh07"
|
||||
$SecurePassword = $smbpass | ConvertTo-SecureString -AsPlainText -Force
|
||||
|
||||
echo "Create user"
|
||||
New-LocalUser $smbuser -Password $SecurePassword -FullName "SMB User" -Description "Account used for smb access."
|
||||
|
||||
echo "Making user admin"
|
||||
Add-LocalGroupMember -Group "Administrators" -Member "$smbuser"
|
||||
|
||||
$path="C:\$smbuser"
|
||||
mkdir $path
|
||||
|
||||
echo "Create share"
|
||||
New-SmbShare -Name $smbuser -Path $path -FullAccess "Administrators" -EncryptData $True
|
||||
|
||||
echo "Grant access to share"
|
||||
Grant-SmbShareAccess -Name $smbuser -AccountName $smbuser -AccessRight Full -Force
|
||||
|
||||
# add $USERPROFILE/tar/bin to path
|
||||
echo $Env:USERPROFILE\tar\bin >> $Env:GITHUB_PATH
|
||||
if: matrix.os == 'windows-latest'
|
||||
|
@ -157,6 +274,7 @@ jobs:
|
|||
- name: Run local Tests
|
||||
env:
|
||||
RESTIC_TEST_FUSE: ${{ matrix.test_fuse }}
|
||||
RESTIC_TEST_SMB: ${{ matrix.test_smb }}
|
||||
run: |
|
||||
go test -cover ${{matrix.test_opts}} ./...
|
||||
|
||||
|
|
|
@ -0,0 +1,7 @@
|
|||
Enhancement: SMB backend: Add SMB backend
|
||||
|
||||
Restic now supports SMB/CIFS backend. You can now add a SMB repository as `-r smb://<host>:<port>/<sharename>/<path>`.
|
||||
|
||||
You can configure the SMB user name (for NTLM authentication) via the environment variable `RESTIC_SMB_USER`, SMB password via the environment variable `RESTIC_SMB_PASSWORD` and optionally SMB domain via the environment variable `RESTIC_SMB_DOMAIN`(default:'WORKGROUP').
|
||||
|
||||
https://github.com/restic/restic/issues/4185
|
|
@ -0,0 +1,10 @@
|
|||
Enhancement: Allow specifying `--host` via environment variable
|
||||
|
||||
Restic commands that operate on snapshots, such as `restic backup` and
|
||||
`restic snapshots`, support the `--host` flag to specify the hostname for
|
||||
grouoping snapshots. They now permit selecting the hostname via the
|
||||
environment variable `RESTIC_HOST`. `--host` still takes precedence over the
|
||||
environment variable.
|
||||
|
||||
https://github.com/restic/restic/issues/4733
|
||||
https://github.com/restic/restic/pull/4734
|
|
@ -0,0 +1,8 @@
|
|||
Bugfix: Fix possible error on concurrent cache cleanup
|
||||
|
||||
If multiple restic processes concurrently cleaned up no longer existing files
|
||||
from the cache, this could cause some of the processes to fail with an `no such
|
||||
file or directory` error. This has been fixed.
|
||||
|
||||
https://github.com/restic/restic/issues/4760
|
||||
https://github.com/restic/restic/pull/4761
|
|
@ -1,89 +1,41 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/signal"
|
||||
"sync"
|
||||
"syscall"
|
||||
|
||||
"github.com/restic/restic/internal/debug"
|
||||
)
|
||||
|
||||
var cleanupHandlers struct {
|
||||
sync.Mutex
|
||||
list []func(code int) (int, error)
|
||||
done bool
|
||||
ch chan os.Signal
|
||||
func createGlobalContext() context.Context {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
ch := make(chan os.Signal, 1)
|
||||
go cleanupHandler(ch, cancel)
|
||||
signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
|
||||
|
||||
return ctx
|
||||
}
|
||||
|
||||
func init() {
|
||||
cleanupHandlers.ch = make(chan os.Signal, 1)
|
||||
go CleanupHandler(cleanupHandlers.ch)
|
||||
signal.Notify(cleanupHandlers.ch, syscall.SIGINT, syscall.SIGTERM)
|
||||
}
|
||||
// cleanupHandler handles the SIGINT and SIGTERM signals.
|
||||
func cleanupHandler(c <-chan os.Signal, cancel context.CancelFunc) {
|
||||
s := <-c
|
||||
debug.Log("signal %v received, cleaning up", s)
|
||||
Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s)
|
||||
|
||||
// AddCleanupHandler adds the function f to the list of cleanup handlers so
|
||||
// that it is executed when all the cleanup handlers are run, e.g. when SIGINT
|
||||
// is received.
|
||||
func AddCleanupHandler(f func(code int) (int, error)) {
|
||||
cleanupHandlers.Lock()
|
||||
defer cleanupHandlers.Unlock()
|
||||
|
||||
// reset the done flag for integration tests
|
||||
cleanupHandlers.done = false
|
||||
|
||||
cleanupHandlers.list = append(cleanupHandlers.list, f)
|
||||
}
|
||||
|
||||
// RunCleanupHandlers runs all registered cleanup handlers
|
||||
func RunCleanupHandlers(code int) int {
|
||||
cleanupHandlers.Lock()
|
||||
defer cleanupHandlers.Unlock()
|
||||
|
||||
if cleanupHandlers.done {
|
||||
return code
|
||||
if val, _ := os.LookupEnv("RESTIC_DEBUG_STACKTRACE_SIGINT"); val != "" {
|
||||
_, _ = os.Stderr.WriteString("\n--- STACKTRACE START ---\n\n")
|
||||
_, _ = os.Stderr.WriteString(debug.DumpStacktrace())
|
||||
_, _ = os.Stderr.WriteString("\n--- STACKTRACE END ---\n")
|
||||
}
|
||||
cleanupHandlers.done = true
|
||||
|
||||
for _, f := range cleanupHandlers.list {
|
||||
var err error
|
||||
code, err = f(code)
|
||||
if err != nil {
|
||||
Warnf("error in cleanup handler: %v\n", err)
|
||||
}
|
||||
}
|
||||
cleanupHandlers.list = nil
|
||||
return code
|
||||
cancel()
|
||||
}
|
||||
|
||||
// CleanupHandler handles the SIGINT and SIGTERM signals.
|
||||
func CleanupHandler(c <-chan os.Signal) {
|
||||
for s := range c {
|
||||
debug.Log("signal %v received, cleaning up", s)
|
||||
Warnf("%ssignal %v received, cleaning up\n", clearLine(0), s)
|
||||
|
||||
if val, _ := os.LookupEnv("RESTIC_DEBUG_STACKTRACE_SIGINT"); val != "" {
|
||||
_, _ = os.Stderr.WriteString("\n--- STACKTRACE START ---\n\n")
|
||||
_, _ = os.Stderr.WriteString(debug.DumpStacktrace())
|
||||
_, _ = os.Stderr.WriteString("\n--- STACKTRACE END ---\n")
|
||||
}
|
||||
|
||||
code := 0
|
||||
|
||||
if s == syscall.SIGINT || s == syscall.SIGTERM {
|
||||
code = 130
|
||||
} else {
|
||||
code = 1
|
||||
}
|
||||
|
||||
Exit(code)
|
||||
}
|
||||
}
|
||||
|
||||
// Exit runs the cleanup handlers and then terminates the process with the
|
||||
// given exit code.
|
||||
// Exit terminates the process with the given exit code.
|
||||
func Exit(code int) {
|
||||
code = RunCleanupHandlers(code)
|
||||
debug.Log("exiting with status code %d", code)
|
||||
os.Exit(code)
|
||||
}
|
||||
|
|
|
@ -114,7 +114,7 @@ func init() {
|
|||
f.BoolVar(&backupOptions.StdinCommand, "stdin-from-command", false, "interpret arguments as command to execute and store its stdout")
|
||||
f.Var(&backupOptions.Tags, "tag", "add `tags` for the new snapshot in the format `tag[,tag,...]` (can be specified multiple times)")
|
||||
f.UintVar(&backupOptions.ReadConcurrency, "read-concurrency", 0, "read `n` files concurrently (default: $RESTIC_READ_CONCURRENCY or 2)")
|
||||
f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually. To prevent an expensive rescan use the \"parent\" flag")
|
||||
f.StringVarP(&backupOptions.Host, "host", "H", "", "set the `hostname` for the snapshot manually (default: $RESTIC_HOST). To prevent an expensive rescan use the \"parent\" flag")
|
||||
f.StringVar(&backupOptions.Host, "hostname", "", "set the `hostname` for the snapshot manually")
|
||||
err := f.MarkDeprecated("hostname", "use --host")
|
||||
if err != nil {
|
||||
|
@ -137,6 +137,11 @@ func init() {
|
|||
// parse read concurrency from env, on error the default value will be used
|
||||
readConcurrency, _ := strconv.ParseUint(os.Getenv("RESTIC_READ_CONCURRENCY"), 10, 32)
|
||||
backupOptions.ReadConcurrency = uint(readConcurrency)
|
||||
|
||||
// parse host from env, if not exists or empty the default value will be used
|
||||
if host := os.Getenv("RESTIC_HOST"); host != "" {
|
||||
backupOptions.Host = host
|
||||
}
|
||||
}
|
||||
|
||||
// filterExisting returns a slice of all existing items, or an error if no
|
||||
|
|
|
@ -199,10 +199,7 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
|
|||
}
|
||||
|
||||
cleanup := prepareCheckCache(opts, &gopts)
|
||||
AddCleanupHandler(func(code int) (int, error) {
|
||||
cleanup()
|
||||
return code, nil
|
||||
})
|
||||
defer cleanup()
|
||||
|
||||
if !gopts.NoLock {
|
||||
Verbosef("create exclusive lock for repository\n")
|
||||
|
@ -222,6 +219,9 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
|
|||
Verbosef("load indexes\n")
|
||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||
hints, errs := chkr.LoadIndex(ctx, bar)
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
errorsFound := false
|
||||
suggestIndexRebuild := false
|
||||
|
@ -283,6 +283,9 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
|
|||
if orphanedPacks > 0 {
|
||||
Verbosef("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
Verbosef("check snapshots, trees and blobs\n")
|
||||
errChan = make(chan error)
|
||||
|
@ -316,9 +319,16 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
|
|||
// Must happen after `errChan` is read from in the above loop to avoid
|
||||
// deadlocking in the case of errors.
|
||||
wg.Wait()
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if opts.CheckUnused {
|
||||
for _, id := range chkr.UnusedBlobs(ctx) {
|
||||
unused, err := chkr.UnusedBlobs(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, id := range unused {
|
||||
Verbosef("unused blob %v\n", id)
|
||||
errorsFound = true
|
||||
}
|
||||
|
@ -395,10 +405,13 @@ func runCheck(ctx context.Context, opts CheckOptions, gopts GlobalOptions, args
|
|||
doReadData(packs)
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if errorsFound {
|
||||
return errors.Fatal("repository contains errors")
|
||||
}
|
||||
|
||||
Verbosef("no errors were found\n")
|
||||
|
||||
return nil
|
||||
|
|
|
@ -53,7 +53,7 @@ func init() {
|
|||
}
|
||||
|
||||
func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []string) error {
|
||||
secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "destination")
|
||||
secondaryGopts, isFromRepo, err := fillSecondaryGlobalOpts(ctx, opts.secondaryRepoOptions, gopts, "destination")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -103,6 +103,9 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
|
|||
// also consider identical snapshot copies
|
||||
dstSnapshotByOriginal[*sn.ID()] = append(dstSnapshotByOriginal[*sn.ID()], sn)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// remember already processed trees across all snapshots
|
||||
visitedTrees := restic.NewIDSet()
|
||||
|
@ -147,7 +150,7 @@ func runCopy(ctx context.Context, opts CopyOptions, gopts GlobalOptions, args []
|
|||
}
|
||||
Verbosef("snapshot %s saved\n", newID.Str())
|
||||
}
|
||||
return nil
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func similarSnapshots(sna *restic.Snapshot, snb *restic.Snapshot) bool {
|
||||
|
|
|
@ -439,7 +439,10 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
|
|||
|
||||
if err != errAllPacksFound {
|
||||
// try to resolve unknown pack ids from the index
|
||||
packIDs = f.indexPacksToBlobs(ctx, packIDs)
|
||||
packIDs, err = f.indexPacksToBlobs(ctx, packIDs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if len(packIDs) > 0 {
|
||||
|
@ -456,13 +459,13 @@ func (f *Finder) packsToBlobs(ctx context.Context, packs []string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struct{}) map[string]struct{} {
|
||||
func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struct{}) (map[string]struct{}, error) {
|
||||
wctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
// remember which packs were found in the index
|
||||
indexPackIDs := make(map[string]struct{})
|
||||
f.repo.Index().Each(wctx, func(pb restic.PackedBlob) {
|
||||
err := f.repo.Index().Each(wctx, func(pb restic.PackedBlob) {
|
||||
idStr := pb.PackID.String()
|
||||
// keep entry in packIDs as Each() returns individual index entries
|
||||
matchingID := false
|
||||
|
@ -481,6 +484,9 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
|
|||
indexPackIDs[idStr] = struct{}{}
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for id := range indexPackIDs {
|
||||
delete(packIDs, id)
|
||||
|
@ -493,7 +499,7 @@ func (f *Finder) indexPacksToBlobs(ctx context.Context, packIDs map[string]struc
|
|||
}
|
||||
Warnf("some pack files are missing from the repository, getting their blobs from the repository index: %v\n\n", list)
|
||||
}
|
||||
return packIDs
|
||||
return packIDs, nil
|
||||
}
|
||||
|
||||
func (f *Finder) findObjectPack(id string, t restic.BlobType) {
|
||||
|
@ -608,6 +614,9 @@ func runFind(ctx context.Context, opts FindOptions, gopts GlobalOptions, args []
|
|||
for sn := range FindFilteredSnapshots(ctx, snapshotLister, repo, &opts.SnapshotFilter, opts.Snapshots) {
|
||||
filteredSnapshots = append(filteredSnapshots, sn)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
sort.Slice(filteredSnapshots, func(i, j int) bool {
|
||||
return filteredSnapshots[i].Time.Before(filteredSnapshots[j].Time)
|
||||
|
|
|
@ -8,6 +8,7 @@ import (
|
|||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui/termstatus"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
|
@ -33,7 +34,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
|
|||
`,
|
||||
DisableAutoGenTag: true,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return runForget(cmd.Context(), forgetOptions, forgetPruneOptions, globalOptions, args)
|
||||
term, cancel := setupTermstatus()
|
||||
defer cancel()
|
||||
return runForget(cmd.Context(), forgetOptions, forgetPruneOptions, globalOptions, term, args)
|
||||
},
|
||||
}
|
||||
|
||||
|
@ -152,7 +155,7 @@ func verifyForgetOptions(opts *ForgetOptions) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOptions, gopts GlobalOptions, args []string) error {
|
||||
func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOptions, gopts GlobalOptions, term *termstatus.Terminal, args []string) error {
|
||||
err := verifyForgetOptions(&opts)
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -173,12 +176,21 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
|
|||
}
|
||||
defer unlock()
|
||||
|
||||
verbosity := gopts.verbosity
|
||||
if gopts.JSON {
|
||||
verbosity = 0
|
||||
}
|
||||
printer := newTerminalProgressPrinter(verbosity, term)
|
||||
|
||||
var snapshots restic.Snapshots
|
||||
removeSnIDs := restic.NewIDSet()
|
||||
|
||||
for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
|
||||
snapshots = append(snapshots, sn)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
var jsonGroups []*ForgetGroup
|
||||
|
||||
|
@ -210,15 +222,11 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
|
|||
}
|
||||
|
||||
if policy.Empty() && len(args) == 0 {
|
||||
if !gopts.JSON {
|
||||
Verbosef("no policy was specified, no snapshots will be removed\n")
|
||||
}
|
||||
printer.P("no policy was specified, no snapshots will be removed\n")
|
||||
}
|
||||
|
||||
if !policy.Empty() {
|
||||
if !gopts.JSON {
|
||||
Verbosef("Applying Policy: %v\n", policy)
|
||||
}
|
||||
printer.P("Applying Policy: %v\n", policy)
|
||||
|
||||
for k, snapshotGroup := range snapshotGroups {
|
||||
if gopts.Verbose >= 1 && !gopts.JSON {
|
||||
|
@ -241,16 +249,16 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
|
|||
keep, remove, reasons := restic.ApplyPolicy(snapshotGroup, policy)
|
||||
|
||||
if len(keep) != 0 && !gopts.Quiet && !gopts.JSON {
|
||||
Printf("keep %d snapshots:\n", len(keep))
|
||||
printer.P("keep %d snapshots:\n", len(keep))
|
||||
PrintSnapshots(globalOptions.stdout, keep, reasons, opts.Compact)
|
||||
Printf("\n")
|
||||
printer.P("\n")
|
||||
}
|
||||
fg.Keep = asJSONSnapshots(keep)
|
||||
|
||||
if len(remove) != 0 && !gopts.Quiet && !gopts.JSON {
|
||||
Printf("remove %d snapshots:\n", len(remove))
|
||||
printer.P("remove %d snapshots:\n", len(remove))
|
||||
PrintSnapshots(globalOptions.stdout, remove, nil, opts.Compact)
|
||||
Printf("\n")
|
||||
printer.P("\n")
|
||||
}
|
||||
fg.Remove = asJSONSnapshots(remove)
|
||||
|
||||
|
@ -265,16 +273,27 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
|
|||
}
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if len(removeSnIDs) > 0 {
|
||||
if !opts.DryRun {
|
||||
err := DeleteFilesChecked(ctx, gopts, repo, removeSnIDs, restic.SnapshotFile)
|
||||
bar := printer.NewCounter("files deleted")
|
||||
err := restic.ParallelRemove(ctx, repo, removeSnIDs, restic.SnapshotFile, func(id restic.ID, err error) error {
|
||||
if err != nil {
|
||||
printer.E("unable to remove %v/%v from the repository\n", restic.SnapshotFile, id)
|
||||
} else {
|
||||
printer.VV("removed %v/%v\n", restic.SnapshotFile, id)
|
||||
}
|
||||
return nil
|
||||
}, bar)
|
||||
bar.Done()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if !gopts.JSON {
|
||||
Printf("Would have removed the following snapshots:\n%v\n\n", removeSnIDs)
|
||||
}
|
||||
printer.P("Would have removed the following snapshots:\n%v\n\n", removeSnIDs)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -286,15 +305,13 @@ func runForget(ctx context.Context, opts ForgetOptions, pruneOptions PruneOption
|
|||
}
|
||||
|
||||
if len(removeSnIDs) > 0 && opts.Prune {
|
||||
if !gopts.JSON {
|
||||
if opts.DryRun {
|
||||
Verbosef("%d snapshots would be removed, running prune dry run\n", len(removeSnIDs))
|
||||
} else {
|
||||
Verbosef("%d snapshots have been removed, running prune\n", len(removeSnIDs))
|
||||
}
|
||||
if opts.DryRun {
|
||||
printer.P("%d snapshots would be removed, running prune dry run\n", len(removeSnIDs))
|
||||
} else {
|
||||
printer.P("%d snapshots have been removed, running prune\n", len(removeSnIDs))
|
||||
}
|
||||
pruneOptions.DryRun = opts.DryRun
|
||||
return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs)
|
||||
return runPruneWithRepo(ctx, pruneOptions, gopts, repo, removeSnIDs, term)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
|
@ -5,6 +5,7 @@ import (
|
|||
"testing"
|
||||
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
"github.com/restic/restic/internal/ui/termstatus"
|
||||
)
|
||||
|
||||
func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
|
||||
|
@ -12,5 +13,7 @@ func testRunForget(t testing.TB, gopts GlobalOptions, args ...string) {
|
|||
pruneOpts := PruneOptions{
|
||||
MaxUnused: "5%",
|
||||
}
|
||||
rtest.OK(t, runForget(context.TODO(), opts, pruneOpts, gopts, args))
|
||||
rtest.OK(t, withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
|
||||
return runForget(context.TODO(), opts, pruneOpts, gopts, term, args)
|
||||
}))
|
||||
}
|
||||
|
|
|
@ -80,7 +80,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
|
|||
return err
|
||||
}
|
||||
|
||||
gopts.password, err = ReadPasswordTwice(gopts,
|
||||
gopts.password, err = ReadPasswordTwice(ctx, gopts,
|
||||
"enter password for new repository: ",
|
||||
"enter password again: ")
|
||||
if err != nil {
|
||||
|
@ -131,7 +131,7 @@ func runInit(ctx context.Context, opts InitOptions, gopts GlobalOptions, args []
|
|||
|
||||
func maybeReadChunkerPolynomial(ctx context.Context, opts InitOptions, gopts GlobalOptions) (*chunker.Pol, error) {
|
||||
if opts.CopyChunkerParameters {
|
||||
otherGopts, _, err := fillSecondaryGlobalOpts(opts.secondaryRepoOptions, gopts, "secondary")
|
||||
otherGopts, _, err := fillSecondaryGlobalOpts(ctx, opts.secondaryRepoOptions, gopts, "secondary")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
@ -60,7 +60,7 @@ func runKeyAdd(ctx context.Context, gopts GlobalOptions, opts KeyAddOptions, arg
|
|||
}
|
||||
|
||||
func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyAddOptions) error {
|
||||
pw, err := getNewPassword(gopts, opts.NewPasswordFile)
|
||||
pw, err := getNewPassword(ctx, gopts, opts.NewPasswordFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -83,7 +83,7 @@ func addKey(ctx context.Context, repo *repository.Repository, gopts GlobalOption
|
|||
// testKeyNewPassword is used to set a new password during integration testing.
|
||||
var testKeyNewPassword string
|
||||
|
||||
func getNewPassword(gopts GlobalOptions, newPasswordFile string) (string, error) {
|
||||
func getNewPassword(ctx context.Context, gopts GlobalOptions, newPasswordFile string) (string, error) {
|
||||
if testKeyNewPassword != "" {
|
||||
return testKeyNewPassword, nil
|
||||
}
|
||||
|
@ -97,7 +97,7 @@ func getNewPassword(gopts GlobalOptions, newPasswordFile string) (string, error)
|
|||
newopts := gopts
|
||||
newopts.password = ""
|
||||
|
||||
return ReadPasswordTwice(newopts,
|
||||
return ReadPasswordTwice(ctx, newopts,
|
||||
"enter new password: ",
|
||||
"enter password again: ")
|
||||
}
|
||||
|
|
|
@ -57,7 +57,7 @@ func runKeyPasswd(ctx context.Context, gopts GlobalOptions, opts KeyPasswdOption
|
|||
}
|
||||
|
||||
func changePassword(ctx context.Context, repo *repository.Repository, gopts GlobalOptions, opts KeyPasswdOptions) error {
|
||||
pw, err := getNewPassword(gopts, opts.NewPasswordFile)
|
||||
pw, err := getNewPassword(ctx, gopts, opts.NewPasswordFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -59,10 +59,9 @@ func runList(ctx context.Context, gopts GlobalOptions, args []string) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
idx.Each(ctx, func(blobs restic.PackedBlob) {
|
||||
return idx.Each(ctx, func(blobs restic.PackedBlob) {
|
||||
Printf("%v %v\n", blobs.Type, blobs.ID)
|
||||
})
|
||||
return nil
|
||||
})
|
||||
default:
|
||||
return errors.Fatal("invalid type")
|
||||
|
|
|
@ -152,28 +152,15 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
|
|||
}
|
||||
}
|
||||
|
||||
AddCleanupHandler(func(code int) (int, error) {
|
||||
debug.Log("running umount cleanup handler for mount at %v", mountpoint)
|
||||
err := umount(mountpoint)
|
||||
if err != nil {
|
||||
Warnf("unable to umount (maybe already umounted or still in use?): %v\n", err)
|
||||
}
|
||||
// replace error code of sigint
|
||||
if code == 130 {
|
||||
code = 0
|
||||
}
|
||||
return code, nil
|
||||
})
|
||||
systemFuse.Debug = func(msg interface{}) {
|
||||
debug.Log("fuse: %v", msg)
|
||||
}
|
||||
|
||||
c, err := systemFuse.Mount(mountpoint, mountOptions...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
systemFuse.Debug = func(msg interface{}) {
|
||||
debug.Log("fuse: %v", msg)
|
||||
}
|
||||
|
||||
cfg := fuse.Config{
|
||||
OwnerIsRoot: opts.OwnerRoot,
|
||||
Filter: opts.SnapshotFilter,
|
||||
|
@ -187,15 +174,26 @@ func runMount(ctx context.Context, opts MountOptions, gopts GlobalOptions, args
|
|||
Printf("When finished, quit with Ctrl-c here or umount the mountpoint.\n")
|
||||
|
||||
debug.Log("serving mount at %v", mountpoint)
|
||||
err = fs.Serve(c, root)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
done := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
defer close(done)
|
||||
err = fs.Serve(c, root)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
debug.Log("running umount cleanup handler for mount at %v", mountpoint)
|
||||
err := systemFuse.Unmount(mountpoint)
|
||||
if err != nil {
|
||||
Warnf("unable to umount (maybe already umounted or still in use?): %v\n", err)
|
||||
}
|
||||
|
||||
return ErrOK
|
||||
case <-done:
|
||||
// clean shutdown, nothing to do
|
||||
}
|
||||
|
||||
<-c.Ready
|
||||
return c.MountError
|
||||
}
|
||||
|
||||
func umount(mountpoint string) error {
|
||||
return systemFuse.Unmount(mountpoint)
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -12,6 +12,7 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
systemFuse "github.com/anacrolix/fuse"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
@ -65,7 +66,7 @@ func testRunMount(t testing.TB, gopts GlobalOptions, dir string, wg *sync.WaitGr
|
|||
func testRunUmount(t testing.TB, dir string) {
|
||||
var err error
|
||||
for i := 0; i < mountWait; i++ {
|
||||
if err = umount(dir); err == nil {
|
||||
if err = systemFuse.Unmount(dir); err == nil {
|
||||
t.Logf("directory %v umounted", dir)
|
||||
return
|
||||
}
|
||||
|
|
|
@ -4,26 +4,20 @@ import (
|
|||
"context"
|
||||
"math"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/index"
|
||||
"github.com/restic/restic/internal/pack"
|
||||
"github.com/restic/restic/internal/repository"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
"github.com/restic/restic/internal/ui/termstatus"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var errorIndexIncomplete = errors.Fatal("index is not complete")
|
||||
var errorPacksMissing = errors.Fatal("packs from index missing in repo")
|
||||
var errorSizeNotMatching = errors.Fatal("pack size does not match calculated size from index")
|
||||
|
||||
var cmdPrune = &cobra.Command{
|
||||
Use: "prune [flags]",
|
||||
Short: "Remove unneeded data from the repository",
|
||||
|
@ -38,7 +32,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
|
|||
`,
|
||||
DisableAutoGenTag: true,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
return runPrune(cmd.Context(), pruneOptions, globalOptions)
|
||||
term, cancel := setupTermstatus()
|
||||
defer cancel()
|
||||
return runPrune(cmd.Context(), pruneOptions, globalOptions, term)
|
||||
},
|
||||
}
|
||||
|
||||
|
@ -138,7 +134,7 @@ func verifyPruneOptions(opts *PruneOptions) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error {
|
||||
func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, term *termstatus.Terminal) error {
|
||||
err := verifyPruneOptions(&opts)
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -154,14 +150,6 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error
|
|||
}
|
||||
defer unlock()
|
||||
|
||||
if repo.Connections() < 2 {
|
||||
return errors.Fatal("prune requires a backend connection limit of at least two")
|
||||
}
|
||||
|
||||
if repo.Config().Version < 2 && opts.RepackUncompressed {
|
||||
return errors.Fatal("compression requires at least repository format version 2")
|
||||
}
|
||||
|
||||
if opts.UnsafeNoSpaceRecovery != "" {
|
||||
repoID := repo.Config().ID
|
||||
if opts.UnsafeNoSpaceRecovery != repoID {
|
||||
|
@ -170,10 +158,10 @@ func runPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions) error
|
|||
opts.unsafeRecovery = true
|
||||
}
|
||||
|
||||
return runPruneWithRepo(ctx, opts, gopts, repo, restic.NewIDSet())
|
||||
return runPruneWithRepo(ctx, opts, gopts, repo, restic.NewIDSet(), term)
|
||||
}
|
||||
|
||||
func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet) error {
|
||||
func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo *repository.Repository, ignoreSnapshots restic.IDSet, term *termstatus.Terminal) error {
|
||||
// we do not need index updates while pruning!
|
||||
repo.DisableAutoIndexUpdate()
|
||||
|
||||
|
@ -181,24 +169,43 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption
|
|||
Print("warning: running prune without a cache, this may be very slow!\n")
|
||||
}
|
||||
|
||||
Verbosef("loading indexes...\n")
|
||||
printer := newTerminalProgressPrinter(gopts.verbosity, term)
|
||||
|
||||
printer.P("loading indexes...\n")
|
||||
// loading the index before the snapshots is ok, as we use an exclusive lock here
|
||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||
bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
|
||||
err := repo.LoadIndex(ctx, bar)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
plan, stats, err := planPrune(ctx, opts, repo, ignoreSnapshots, gopts.Quiet)
|
||||
popts := repository.PruneOptions{
|
||||
DryRun: opts.DryRun,
|
||||
UnsafeRecovery: opts.unsafeRecovery,
|
||||
|
||||
MaxUnusedBytes: opts.maxUnusedBytes,
|
||||
MaxRepackBytes: opts.MaxRepackBytes,
|
||||
|
||||
RepackCachableOnly: opts.RepackCachableOnly,
|
||||
RepackSmall: opts.RepackSmall,
|
||||
RepackUncompressed: opts.RepackUncompressed,
|
||||
}
|
||||
|
||||
plan, err := repository.PlanPrune(ctx, popts, repo, func(ctx context.Context, repo restic.Repository) (usedBlobs restic.CountedBlobSet, err error) {
|
||||
return getUsedBlobs(ctx, repo, ignoreSnapshots, printer)
|
||||
}, printer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if opts.DryRun {
|
||||
Verbosef("\nWould have made the following changes:")
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
err = printPruneStats(stats)
|
||||
if popts.DryRun {
|
||||
printer.P("\nWould have made the following changes:")
|
||||
}
|
||||
|
||||
err = printPruneStats(printer, plan.Stats())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -206,605 +213,54 @@ func runPruneWithRepo(ctx context.Context, opts PruneOptions, gopts GlobalOption
|
|||
// Trigger GC to reset garbage collection threshold
|
||||
runtime.GC()
|
||||
|
||||
return doPrune(ctx, opts, gopts, repo, plan)
|
||||
}
|
||||
|
||||
type pruneStats struct {
|
||||
blobs struct {
|
||||
used uint
|
||||
duplicate uint
|
||||
unused uint
|
||||
remove uint
|
||||
repack uint
|
||||
repackrm uint
|
||||
}
|
||||
size struct {
|
||||
used uint64
|
||||
duplicate uint64
|
||||
unused uint64
|
||||
remove uint64
|
||||
repack uint64
|
||||
repackrm uint64
|
||||
unref uint64
|
||||
uncompressed uint64
|
||||
}
|
||||
packs struct {
|
||||
used uint
|
||||
unused uint
|
||||
partlyUsed uint
|
||||
unref uint
|
||||
keep uint
|
||||
repack uint
|
||||
remove uint
|
||||
}
|
||||
}
|
||||
|
||||
type prunePlan struct {
|
||||
removePacksFirst restic.IDSet // packs to remove first (unreferenced packs)
|
||||
repackPacks restic.IDSet // packs to repack
|
||||
keepBlobs restic.CountedBlobSet // blobs to keep during repacking
|
||||
removePacks restic.IDSet // packs to remove
|
||||
ignorePacks restic.IDSet // packs to ignore when rebuilding the index
|
||||
}
|
||||
|
||||
type packInfo struct {
|
||||
usedBlobs uint
|
||||
unusedBlobs uint
|
||||
usedSize uint64
|
||||
unusedSize uint64
|
||||
tpe restic.BlobType
|
||||
uncompressed bool
|
||||
}
|
||||
|
||||
type packInfoWithID struct {
|
||||
ID restic.ID
|
||||
packInfo
|
||||
mustCompress bool
|
||||
}
|
||||
|
||||
// planPrune selects which files to rewrite and which to delete and which blobs to keep.
|
||||
// Also some summary statistics are returned.
|
||||
func planPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (prunePlan, pruneStats, error) {
|
||||
var stats pruneStats
|
||||
|
||||
usedBlobs, err := getUsedBlobs(ctx, repo, ignoreSnapshots, quiet)
|
||||
if err != nil {
|
||||
return prunePlan{}, stats, err
|
||||
}
|
||||
|
||||
Verbosef("searching used packs...\n")
|
||||
keepBlobs, indexPack, err := packInfoFromIndex(ctx, repo.Index(), usedBlobs, &stats)
|
||||
if err != nil {
|
||||
return prunePlan{}, stats, err
|
||||
}
|
||||
|
||||
Verbosef("collecting packs for deletion and repacking\n")
|
||||
plan, err := decidePackAction(ctx, opts, repo, indexPack, &stats, quiet)
|
||||
if err != nil {
|
||||
return prunePlan{}, stats, err
|
||||
}
|
||||
|
||||
if len(plan.repackPacks) != 0 {
|
||||
blobCount := keepBlobs.Len()
|
||||
// when repacking, we do not want to keep blobs which are
|
||||
// already contained in kept packs, so delete them from keepBlobs
|
||||
repo.Index().Each(ctx, func(blob restic.PackedBlob) {
|
||||
if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) {
|
||||
return
|
||||
}
|
||||
keepBlobs.Delete(blob.BlobHandle)
|
||||
})
|
||||
|
||||
if keepBlobs.Len() < blobCount/2 {
|
||||
// replace with copy to shrink map to necessary size if there's a chance to benefit
|
||||
keepBlobs = keepBlobs.Copy()
|
||||
}
|
||||
} else {
|
||||
// keepBlobs is only needed if packs are repacked
|
||||
keepBlobs = nil
|
||||
}
|
||||
plan.keepBlobs = keepBlobs
|
||||
|
||||
return plan, stats, nil
|
||||
}
|
||||
|
||||
func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs restic.CountedBlobSet, stats *pruneStats) (restic.CountedBlobSet, map[restic.ID]packInfo, error) {
|
||||
// iterate over all blobs in index to find out which blobs are duplicates
|
||||
// The counter in usedBlobs describes how many instances of the blob exist in the repository index
|
||||
// Thus 0 == blob is missing, 1 == blob exists once, >= 2 == duplicates exist
|
||||
idx.Each(ctx, func(blob restic.PackedBlob) {
|
||||
bh := blob.BlobHandle
|
||||
count, ok := usedBlobs[bh]
|
||||
if ok {
|
||||
if count < math.MaxUint8 {
|
||||
// don't overflow, but saturate count at 255
|
||||
// this can lead to a non-optimal pack selection, but won't cause
|
||||
// problems otherwise
|
||||
count++
|
||||
}
|
||||
|
||||
usedBlobs[bh] = count
|
||||
}
|
||||
})
|
||||
|
||||
// Check if all used blobs have been found in index
|
||||
missingBlobs := restic.NewBlobSet()
|
||||
for bh, count := range usedBlobs {
|
||||
if count == 0 {
|
||||
// blob does not exist in any pack files
|
||||
missingBlobs.Insert(bh)
|
||||
}
|
||||
}
|
||||
|
||||
if len(missingBlobs) != 0 {
|
||||
Warnf("%v not found in the index\n\n"+
|
||||
"Integrity check failed: Data seems to be missing.\n"+
|
||||
"Will not start prune to prevent (additional) data loss!\n"+
|
||||
"Please report this error (along with the output of the 'prune' run) at\n"+
|
||||
"https://github.com/restic/restic/issues/new/choose\n", missingBlobs)
|
||||
return nil, nil, errorIndexIncomplete
|
||||
}
|
||||
|
||||
indexPack := make(map[restic.ID]packInfo)
|
||||
|
||||
// save computed pack header size
|
||||
for pid, hdrSize := range pack.Size(ctx, idx, true) {
|
||||
// initialize tpe with NumBlobTypes to indicate it's not set
|
||||
indexPack[pid] = packInfo{tpe: restic.NumBlobTypes, usedSize: uint64(hdrSize)}
|
||||
}
|
||||
|
||||
hasDuplicates := false
|
||||
// iterate over all blobs in index to generate packInfo
|
||||
idx.Each(ctx, func(blob restic.PackedBlob) {
|
||||
ip := indexPack[blob.PackID]
|
||||
|
||||
// Set blob type if not yet set
|
||||
if ip.tpe == restic.NumBlobTypes {
|
||||
ip.tpe = blob.Type
|
||||
}
|
||||
|
||||
// mark mixed packs with "Invalid blob type"
|
||||
if ip.tpe != blob.Type {
|
||||
ip.tpe = restic.InvalidBlob
|
||||
}
|
||||
|
||||
bh := blob.BlobHandle
|
||||
size := uint64(blob.Length)
|
||||
dupCount := usedBlobs[bh]
|
||||
switch {
|
||||
case dupCount >= 2:
|
||||
hasDuplicates = true
|
||||
// mark as unused for now, we will later on select one copy
|
||||
ip.unusedSize += size
|
||||
ip.unusedBlobs++
|
||||
|
||||
// count as duplicate, will later on change one copy to be counted as used
|
||||
stats.size.duplicate += size
|
||||
stats.blobs.duplicate++
|
||||
case dupCount == 1: // used blob, not duplicate
|
||||
ip.usedSize += size
|
||||
ip.usedBlobs++
|
||||
|
||||
stats.size.used += size
|
||||
stats.blobs.used++
|
||||
default: // unused blob
|
||||
ip.unusedSize += size
|
||||
ip.unusedBlobs++
|
||||
|
||||
stats.size.unused += size
|
||||
stats.blobs.unused++
|
||||
}
|
||||
if !blob.IsCompressed() {
|
||||
ip.uncompressed = true
|
||||
}
|
||||
// update indexPack
|
||||
indexPack[blob.PackID] = ip
|
||||
})
|
||||
|
||||
// if duplicate blobs exist, those will be set to either "used" or "unused":
|
||||
// - mark only one occurrence of duplicate blobs as used
|
||||
// - if there are already some used blobs in a pack, possibly mark duplicates in this pack as "used"
|
||||
// - if there are no used blobs in a pack, possibly mark duplicates as "unused"
|
||||
if hasDuplicates {
|
||||
// iterate again over all blobs in index (this is pretty cheap, all in-mem)
|
||||
idx.Each(ctx, func(blob restic.PackedBlob) {
|
||||
bh := blob.BlobHandle
|
||||
count, ok := usedBlobs[bh]
|
||||
// skip non-duplicate, aka. normal blobs
|
||||
// count == 0 is used to mark that this was a duplicate blob with only a single occurrence remaining
|
||||
if !ok || count == 1 {
|
||||
return
|
||||
}
|
||||
|
||||
ip := indexPack[blob.PackID]
|
||||
size := uint64(blob.Length)
|
||||
switch {
|
||||
case ip.usedBlobs > 0, count == 0:
|
||||
// other used blobs in pack or "last" occurrence -> transition to used
|
||||
ip.usedSize += size
|
||||
ip.usedBlobs++
|
||||
ip.unusedSize -= size
|
||||
ip.unusedBlobs--
|
||||
// same for the global statistics
|
||||
stats.size.used += size
|
||||
stats.blobs.used++
|
||||
stats.size.duplicate -= size
|
||||
stats.blobs.duplicate--
|
||||
// let other occurrences remain marked as unused
|
||||
usedBlobs[bh] = 1
|
||||
default:
|
||||
// remain unused and decrease counter
|
||||
count--
|
||||
if count == 1 {
|
||||
// setting count to 1 would lead to forgetting that this blob had duplicates
|
||||
// thus use the special value zero. This will select the last instance of the blob for keeping.
|
||||
count = 0
|
||||
}
|
||||
usedBlobs[bh] = count
|
||||
}
|
||||
// update indexPack
|
||||
indexPack[blob.PackID] = ip
|
||||
})
|
||||
}
|
||||
|
||||
// Sanity check. If no duplicates exist, all blobs have value 1. After handling
|
||||
// duplicates, this also applies to duplicates.
|
||||
for _, count := range usedBlobs {
|
||||
if count != 1 {
|
||||
panic("internal error during blob selection")
|
||||
}
|
||||
}
|
||||
|
||||
return usedBlobs, indexPack, nil
|
||||
}
|
||||
|
||||
func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Repository, indexPack map[restic.ID]packInfo, stats *pruneStats, quiet bool) (prunePlan, error) {
|
||||
removePacksFirst := restic.NewIDSet()
|
||||
removePacks := restic.NewIDSet()
|
||||
repackPacks := restic.NewIDSet()
|
||||
|
||||
var repackCandidates []packInfoWithID
|
||||
var repackSmallCandidates []packInfoWithID
|
||||
repoVersion := repo.Config().Version
|
||||
// only repack very small files by default
|
||||
targetPackSize := repo.PackSize() / 25
|
||||
if opts.RepackSmall {
|
||||
// consider files with at least 80% of the target size as large enough
|
||||
targetPackSize = repo.PackSize() / 5 * 4
|
||||
}
|
||||
|
||||
// loop over all packs and decide what to do
|
||||
bar := newProgressMax(!quiet, uint64(len(indexPack)), "packs processed")
|
||||
err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
|
||||
p, ok := indexPack[id]
|
||||
if !ok {
|
||||
// Pack was not referenced in index and is not used => immediately remove!
|
||||
Verboseff("will remove pack %v as it is unused and not indexed\n", id.Str())
|
||||
removePacksFirst.Insert(id)
|
||||
stats.size.unref += uint64(packSize)
|
||||
return nil
|
||||
}
|
||||
|
||||
if p.unusedSize+p.usedSize != uint64(packSize) && p.usedBlobs != 0 {
|
||||
// Pack size does not fit and pack is needed => error
|
||||
// If the pack is not needed, this is no error, the pack can
|
||||
// and will be simply removed, see below.
|
||||
Warnf("pack %s: calculated size %d does not match real size %d\nRun 'restic repair index'.\n",
|
||||
id.Str(), p.unusedSize+p.usedSize, packSize)
|
||||
return errorSizeNotMatching
|
||||
}
|
||||
|
||||
// statistics
|
||||
switch {
|
||||
case p.usedBlobs == 0:
|
||||
stats.packs.unused++
|
||||
case p.unusedBlobs == 0:
|
||||
stats.packs.used++
|
||||
default:
|
||||
stats.packs.partlyUsed++
|
||||
}
|
||||
|
||||
if p.uncompressed {
|
||||
stats.size.uncompressed += p.unusedSize + p.usedSize
|
||||
}
|
||||
mustCompress := false
|
||||
if repoVersion >= 2 {
|
||||
// repo v2: always repack tree blobs if uncompressed
|
||||
// compress data blobs if requested
|
||||
mustCompress = (p.tpe == restic.TreeBlob || opts.RepackUncompressed) && p.uncompressed
|
||||
}
|
||||
|
||||
// decide what to do
|
||||
switch {
|
||||
case p.usedBlobs == 0:
|
||||
// All blobs in pack are no longer used => remove pack!
|
||||
removePacks.Insert(id)
|
||||
stats.blobs.remove += p.unusedBlobs
|
||||
stats.size.remove += p.unusedSize
|
||||
|
||||
case opts.RepackCachableOnly && p.tpe == restic.DataBlob:
|
||||
// if this is a data pack and --repack-cacheable-only is set => keep pack!
|
||||
stats.packs.keep++
|
||||
|
||||
case p.unusedBlobs == 0 && p.tpe != restic.InvalidBlob && !mustCompress:
|
||||
if packSize >= int64(targetPackSize) {
|
||||
// All blobs in pack are used and not mixed => keep pack!
|
||||
stats.packs.keep++
|
||||
} else {
|
||||
repackSmallCandidates = append(repackSmallCandidates, packInfoWithID{ID: id, packInfo: p, mustCompress: mustCompress})
|
||||
}
|
||||
|
||||
default:
|
||||
// all other packs are candidates for repacking
|
||||
repackCandidates = append(repackCandidates, packInfoWithID{ID: id, packInfo: p, mustCompress: mustCompress})
|
||||
}
|
||||
|
||||
delete(indexPack, id)
|
||||
bar.Add(1)
|
||||
return nil
|
||||
})
|
||||
bar.Done()
|
||||
if err != nil {
|
||||
return prunePlan{}, err
|
||||
}
|
||||
|
||||
// At this point indexPacks contains only missing packs!
|
||||
|
||||
// missing packs that are not needed can be ignored
|
||||
ignorePacks := restic.NewIDSet()
|
||||
for id, p := range indexPack {
|
||||
if p.usedBlobs == 0 {
|
||||
ignorePacks.Insert(id)
|
||||
stats.blobs.remove += p.unusedBlobs
|
||||
stats.size.remove += p.unusedSize
|
||||
delete(indexPack, id)
|
||||
}
|
||||
}
|
||||
|
||||
if len(indexPack) != 0 {
|
||||
Warnf("The index references %d needed pack files which are missing from the repository:\n", len(indexPack))
|
||||
for id := range indexPack {
|
||||
Warnf(" %v\n", id)
|
||||
}
|
||||
return prunePlan{}, errorPacksMissing
|
||||
}
|
||||
if len(ignorePacks) != 0 {
|
||||
Warnf("Missing but unneeded pack files are referenced in the index, will be repaired\n")
|
||||
for id := range ignorePacks {
|
||||
Warnf("will forget missing pack file %v\n", id)
|
||||
}
|
||||
}
|
||||
|
||||
if len(repackSmallCandidates) < 10 {
|
||||
// too few small files to be worth the trouble, this also prevents endlessly repacking
|
||||
// if there is just a single pack file below the target size
|
||||
stats.packs.keep += uint(len(repackSmallCandidates))
|
||||
} else {
|
||||
repackCandidates = append(repackCandidates, repackSmallCandidates...)
|
||||
}
|
||||
|
||||
// Sort repackCandidates such that packs with highest ratio unused/used space are picked first.
|
||||
// This is equivalent to sorting by unused / total space.
|
||||
// Instead of unused[i] / used[i] > unused[j] / used[j] we use
|
||||
// unused[i] * used[j] > unused[j] * used[i] as uint32*uint32 < uint64
|
||||
// Moreover packs containing trees and too small packs are sorted to the beginning
|
||||
sort.Slice(repackCandidates, func(i, j int) bool {
|
||||
pi := repackCandidates[i].packInfo
|
||||
pj := repackCandidates[j].packInfo
|
||||
switch {
|
||||
case pi.tpe != restic.DataBlob && pj.tpe == restic.DataBlob:
|
||||
return true
|
||||
case pj.tpe != restic.DataBlob && pi.tpe == restic.DataBlob:
|
||||
return false
|
||||
case pi.unusedSize+pi.usedSize < uint64(targetPackSize) && pj.unusedSize+pj.usedSize >= uint64(targetPackSize):
|
||||
return true
|
||||
case pj.unusedSize+pj.usedSize < uint64(targetPackSize) && pi.unusedSize+pi.usedSize >= uint64(targetPackSize):
|
||||
return false
|
||||
}
|
||||
return pi.unusedSize*pj.usedSize > pj.unusedSize*pi.usedSize
|
||||
})
|
||||
|
||||
repack := func(id restic.ID, p packInfo) {
|
||||
repackPacks.Insert(id)
|
||||
stats.blobs.repack += p.unusedBlobs + p.usedBlobs
|
||||
stats.size.repack += p.unusedSize + p.usedSize
|
||||
stats.blobs.repackrm += p.unusedBlobs
|
||||
stats.size.repackrm += p.unusedSize
|
||||
if p.uncompressed {
|
||||
stats.size.uncompressed -= p.unusedSize + p.usedSize
|
||||
}
|
||||
}
|
||||
|
||||
// calculate limit for number of unused bytes in the repo after repacking
|
||||
maxUnusedSizeAfter := opts.maxUnusedBytes(stats.size.used)
|
||||
|
||||
for _, p := range repackCandidates {
|
||||
reachedUnusedSizeAfter := (stats.size.unused-stats.size.remove-stats.size.repackrm < maxUnusedSizeAfter)
|
||||
reachedRepackSize := stats.size.repack+p.unusedSize+p.usedSize >= opts.MaxRepackBytes
|
||||
packIsLargeEnough := p.unusedSize+p.usedSize >= uint64(targetPackSize)
|
||||
|
||||
switch {
|
||||
case reachedRepackSize:
|
||||
stats.packs.keep++
|
||||
|
||||
case p.tpe != restic.DataBlob, p.mustCompress:
|
||||
// repacking non-data packs / uncompressed-trees is only limited by repackSize
|
||||
repack(p.ID, p.packInfo)
|
||||
|
||||
case reachedUnusedSizeAfter && packIsLargeEnough:
|
||||
// for all other packs stop repacking if tolerated unused size is reached.
|
||||
stats.packs.keep++
|
||||
|
||||
default:
|
||||
repack(p.ID, p.packInfo)
|
||||
}
|
||||
}
|
||||
|
||||
stats.packs.unref = uint(len(removePacksFirst))
|
||||
stats.packs.repack = uint(len(repackPacks))
|
||||
stats.packs.remove = uint(len(removePacks))
|
||||
|
||||
if repo.Config().Version < 2 {
|
||||
// compression not supported for repository format version 1
|
||||
stats.size.uncompressed = 0
|
||||
}
|
||||
|
||||
return prunePlan{removePacksFirst: removePacksFirst,
|
||||
removePacks: removePacks,
|
||||
repackPacks: repackPacks,
|
||||
ignorePacks: ignorePacks,
|
||||
}, nil
|
||||
return plan.Execute(ctx, printer)
|
||||
}
|
||||
|
||||
// printPruneStats prints out the statistics
|
||||
func printPruneStats(stats pruneStats) error {
|
||||
Verboseff("\nused: %10d blobs / %s\n", stats.blobs.used, ui.FormatBytes(stats.size.used))
|
||||
if stats.blobs.duplicate > 0 {
|
||||
Verboseff("duplicates: %10d blobs / %s\n", stats.blobs.duplicate, ui.FormatBytes(stats.size.duplicate))
|
||||
func printPruneStats(printer progress.Printer, stats repository.PruneStats) error {
|
||||
printer.V("\nused: %10d blobs / %s\n", stats.Blobs.Used, ui.FormatBytes(stats.Size.Used))
|
||||
if stats.Blobs.Duplicate > 0 {
|
||||
printer.V("duplicates: %10d blobs / %s\n", stats.Blobs.Duplicate, ui.FormatBytes(stats.Size.Duplicate))
|
||||
}
|
||||
Verboseff("unused: %10d blobs / %s\n", stats.blobs.unused, ui.FormatBytes(stats.size.unused))
|
||||
if stats.size.unref > 0 {
|
||||
Verboseff("unreferenced: %s\n", ui.FormatBytes(stats.size.unref))
|
||||
printer.V("unused: %10d blobs / %s\n", stats.Blobs.Unused, ui.FormatBytes(stats.Size.Unused))
|
||||
if stats.Size.Unref > 0 {
|
||||
printer.V("unreferenced: %s\n", ui.FormatBytes(stats.Size.Unref))
|
||||
}
|
||||
totalBlobs := stats.blobs.used + stats.blobs.unused + stats.blobs.duplicate
|
||||
totalSize := stats.size.used + stats.size.duplicate + stats.size.unused + stats.size.unref
|
||||
unusedSize := stats.size.duplicate + stats.size.unused
|
||||
Verboseff("total: %10d blobs / %s\n", totalBlobs, ui.FormatBytes(totalSize))
|
||||
Verboseff("unused size: %s of total size\n", ui.FormatPercent(unusedSize, totalSize))
|
||||
totalBlobs := stats.Blobs.Used + stats.Blobs.Unused + stats.Blobs.Duplicate
|
||||
totalSize := stats.Size.Used + stats.Size.Duplicate + stats.Size.Unused + stats.Size.Unref
|
||||
unusedSize := stats.Size.Duplicate + stats.Size.Unused
|
||||
printer.V("total: %10d blobs / %s\n", totalBlobs, ui.FormatBytes(totalSize))
|
||||
printer.V("unused size: %s of total size\n", ui.FormatPercent(unusedSize, totalSize))
|
||||
|
||||
Verbosef("\nto repack: %10d blobs / %s\n", stats.blobs.repack, ui.FormatBytes(stats.size.repack))
|
||||
Verbosef("this removes: %10d blobs / %s\n", stats.blobs.repackrm, ui.FormatBytes(stats.size.repackrm))
|
||||
Verbosef("to delete: %10d blobs / %s\n", stats.blobs.remove, ui.FormatBytes(stats.size.remove+stats.size.unref))
|
||||
totalPruneSize := stats.size.remove + stats.size.repackrm + stats.size.unref
|
||||
Verbosef("total prune: %10d blobs / %s\n", stats.blobs.remove+stats.blobs.repackrm, ui.FormatBytes(totalPruneSize))
|
||||
if stats.size.uncompressed > 0 {
|
||||
Verbosef("not yet compressed: %s\n", ui.FormatBytes(stats.size.uncompressed))
|
||||
printer.P("\nto repack: %10d blobs / %s\n", stats.Blobs.Repack, ui.FormatBytes(stats.Size.Repack))
|
||||
printer.P("this removes: %10d blobs / %s\n", stats.Blobs.Repackrm, ui.FormatBytes(stats.Size.Repackrm))
|
||||
printer.P("to delete: %10d blobs / %s\n", stats.Blobs.Remove, ui.FormatBytes(stats.Size.Remove+stats.Size.Unref))
|
||||
totalPruneSize := stats.Size.Remove + stats.Size.Repackrm + stats.Size.Unref
|
||||
printer.P("total prune: %10d blobs / %s\n", stats.Blobs.Remove+stats.Blobs.Repackrm, ui.FormatBytes(totalPruneSize))
|
||||
if stats.Size.Uncompressed > 0 {
|
||||
printer.P("not yet compressed: %s\n", ui.FormatBytes(stats.Size.Uncompressed))
|
||||
}
|
||||
Verbosef("remaining: %10d blobs / %s\n", totalBlobs-(stats.blobs.remove+stats.blobs.repackrm), ui.FormatBytes(totalSize-totalPruneSize))
|
||||
unusedAfter := unusedSize - stats.size.remove - stats.size.repackrm
|
||||
Verbosef("unused size after prune: %s (%s of remaining size)\n",
|
||||
printer.P("remaining: %10d blobs / %s\n", totalBlobs-(stats.Blobs.Remove+stats.Blobs.Repackrm), ui.FormatBytes(totalSize-totalPruneSize))
|
||||
unusedAfter := unusedSize - stats.Size.Remove - stats.Size.Repackrm
|
||||
printer.P("unused size after prune: %s (%s of remaining size)\n",
|
||||
ui.FormatBytes(unusedAfter), ui.FormatPercent(unusedAfter, totalSize-totalPruneSize))
|
||||
Verbosef("\n")
|
||||
Verboseff("totally used packs: %10d\n", stats.packs.used)
|
||||
Verboseff("partly used packs: %10d\n", stats.packs.partlyUsed)
|
||||
Verboseff("unused packs: %10d\n\n", stats.packs.unused)
|
||||
printer.P("\n")
|
||||
printer.V("totally used packs: %10d\n", stats.Packs.Used)
|
||||
printer.V("partly used packs: %10d\n", stats.Packs.PartlyUsed)
|
||||
printer.V("unused packs: %10d\n\n", stats.Packs.Unused)
|
||||
|
||||
Verboseff("to keep: %10d packs\n", stats.packs.keep)
|
||||
Verboseff("to repack: %10d packs\n", stats.packs.repack)
|
||||
Verboseff("to delete: %10d packs\n", stats.packs.remove)
|
||||
if stats.packs.unref > 0 {
|
||||
Verboseff("to delete: %10d unreferenced packs\n\n", stats.packs.unref)
|
||||
printer.V("to keep: %10d packs\n", stats.Packs.Keep)
|
||||
printer.V("to repack: %10d packs\n", stats.Packs.Repack)
|
||||
printer.V("to delete: %10d packs\n", stats.Packs.Remove)
|
||||
if stats.Packs.Unref > 0 {
|
||||
printer.V("to delete: %10d unreferenced packs\n\n", stats.Packs.Unref)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// doPrune does the actual pruning:
|
||||
// - remove unreferenced packs first
|
||||
// - repack given pack files while keeping the given blobs
|
||||
// - rebuild the index while ignoring all files that will be deleted
|
||||
// - delete the files
|
||||
// plan.removePacks and plan.ignorePacks are modified in this function.
|
||||
func doPrune(ctx context.Context, opts PruneOptions, gopts GlobalOptions, repo restic.Repository, plan prunePlan) (err error) {
|
||||
if opts.DryRun {
|
||||
if !gopts.JSON && gopts.verbosity >= 2 {
|
||||
Printf("Repeated prune dry-runs can report slightly different amounts of data to keep or repack. This is expected behavior.\n\n")
|
||||
if len(plan.removePacksFirst) > 0 {
|
||||
Printf("Would have removed the following unreferenced packs:\n%v\n\n", plan.removePacksFirst)
|
||||
}
|
||||
Printf("Would have repacked and removed the following packs:\n%v\n\n", plan.repackPacks)
|
||||
Printf("Would have removed the following no longer used packs:\n%v\n\n", plan.removePacks)
|
||||
}
|
||||
// Always quit here if DryRun was set!
|
||||
return nil
|
||||
}
|
||||
|
||||
// unreferenced packs can be safely deleted first
|
||||
if len(plan.removePacksFirst) != 0 {
|
||||
Verbosef("deleting unreferenced packs\n")
|
||||
DeleteFiles(ctx, gopts, repo, plan.removePacksFirst, restic.PackFile)
|
||||
}
|
||||
|
||||
if len(plan.repackPacks) != 0 {
|
||||
Verbosef("repacking packs\n")
|
||||
bar := newProgressMax(!gopts.Quiet, uint64(len(plan.repackPacks)), "packs repacked")
|
||||
_, err := repository.Repack(ctx, repo, repo, plan.repackPacks, plan.keepBlobs, bar)
|
||||
bar.Done()
|
||||
if err != nil {
|
||||
return errors.Fatal(err.Error())
|
||||
}
|
||||
|
||||
// Also remove repacked packs
|
||||
plan.removePacks.Merge(plan.repackPacks)
|
||||
|
||||
if len(plan.keepBlobs) != 0 {
|
||||
Warnf("%v was not repacked\n\n"+
|
||||
"Integrity check failed.\n"+
|
||||
"Please report this error (along with the output of the 'prune' run) at\n"+
|
||||
"https://github.com/restic/restic/issues/new/choose\n", plan.keepBlobs)
|
||||
return errors.Fatal("internal error: blobs were not repacked")
|
||||
}
|
||||
|
||||
// allow GC of the blob set
|
||||
plan.keepBlobs = nil
|
||||
}
|
||||
|
||||
if len(plan.ignorePacks) == 0 {
|
||||
plan.ignorePacks = plan.removePacks
|
||||
} else {
|
||||
plan.ignorePacks.Merge(plan.removePacks)
|
||||
}
|
||||
|
||||
if opts.unsafeRecovery {
|
||||
Verbosef("deleting index files\n")
|
||||
indexFiles := repo.Index().(*index.MasterIndex).IDs()
|
||||
err = DeleteFilesChecked(ctx, gopts, repo, indexFiles, restic.IndexFile)
|
||||
if err != nil {
|
||||
return errors.Fatalf("%s", err)
|
||||
}
|
||||
} else if len(plan.ignorePacks) != 0 {
|
||||
err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil, false)
|
||||
if err != nil {
|
||||
return errors.Fatalf("%s", err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(plan.removePacks) != 0 {
|
||||
Verbosef("removing %d old packs\n", len(plan.removePacks))
|
||||
DeleteFiles(ctx, gopts, repo, plan.removePacks, restic.PackFile)
|
||||
}
|
||||
|
||||
if opts.unsafeRecovery {
|
||||
err = rebuildIndexFiles(ctx, gopts, repo, plan.ignorePacks, nil, true)
|
||||
if err != nil {
|
||||
return errors.Fatalf("%s", err)
|
||||
}
|
||||
}
|
||||
|
||||
Verbosef("done\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
func rebuildIndexFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs, skipDeletion bool) error {
|
||||
Verbosef("rebuilding index\n")
|
||||
|
||||
bar := newProgressMax(!gopts.Quiet, 0, "packs processed")
|
||||
return repo.Index().Save(ctx, repo, removePacks, extraObsolete, restic.MasterIndexSaveOpts{
|
||||
SaveProgress: bar,
|
||||
DeleteProgress: func() *progress.Counter {
|
||||
return newProgressMax(!gopts.Quiet, 0, "old indexes deleted")
|
||||
},
|
||||
DeleteReport: func(id restic.ID, _ error) {
|
||||
if gopts.verbosity > 2 {
|
||||
Verbosef("removed index %v\n", id.String())
|
||||
}
|
||||
},
|
||||
SkipDeletion: skipDeletion,
|
||||
})
|
||||
}
|
||||
|
||||
func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, quiet bool) (usedBlobs restic.CountedBlobSet, err error) {
|
||||
func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots restic.IDSet, printer progress.Printer) (usedBlobs restic.CountedBlobSet, err error) {
|
||||
var snapshotTrees restic.IDs
|
||||
Verbosef("loading all snapshots...\n")
|
||||
printer.P("loading all snapshots...\n")
|
||||
err = restic.ForAllSnapshots(ctx, repo, repo, ignoreSnapshots,
|
||||
func(id restic.ID, sn *restic.Snapshot, err error) error {
|
||||
if err != nil {
|
||||
|
@ -819,11 +275,12 @@ func getUsedBlobs(ctx context.Context, repo restic.Repository, ignoreSnapshots r
|
|||
return nil, errors.Fatalf("failed loading snapshot: %v", err)
|
||||
}
|
||||
|
||||
Verbosef("finding data that is still in use for %d snapshots\n", len(snapshotTrees))
|
||||
printer.P("finding data that is still in use for %d snapshots\n", len(snapshotTrees))
|
||||
|
||||
usedBlobs = restic.NewCountedBlobSet()
|
||||
|
||||
bar := newProgressMax(!quiet, uint64(len(snapshotTrees)), "snapshots")
|
||||
bar := printer.NewCounter("snapshots")
|
||||
bar.SetMax(uint64(len(snapshotTrees)))
|
||||
defer bar.Done()
|
||||
|
||||
err = restic.FindUsedBlobs(ctx, repo, snapshotTrees, usedBlobs, bar)
|
||||
|
|
|
@ -7,7 +7,9 @@ import (
|
|||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/backend"
|
||||
"github.com/restic/restic/internal/repository"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
"github.com/restic/restic/internal/ui/termstatus"
|
||||
)
|
||||
|
||||
func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
|
||||
|
@ -16,7 +18,9 @@ func testRunPrune(t testing.TB, gopts GlobalOptions, opts PruneOptions) {
|
|||
defer func() {
|
||||
gopts.backendTestHook = oldHook
|
||||
}()
|
||||
rtest.OK(t, runPrune(context.TODO(), opts, gopts))
|
||||
rtest.OK(t, withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
|
||||
return runPrune(context.TODO(), opts, gopts, term)
|
||||
}))
|
||||
}
|
||||
|
||||
func TestPrune(t *testing.T) {
|
||||
|
@ -31,7 +35,7 @@ func testPruneVariants(t *testing.T, unsafeNoSpaceRecovery bool) {
|
|||
}
|
||||
t.Run("0"+suffix, func(t *testing.T) {
|
||||
opts := PruneOptions{MaxUnused: "0%", unsafeRecovery: unsafeNoSpaceRecovery}
|
||||
checkOpts := CheckOptions{ReadData: true, CheckUnused: true}
|
||||
checkOpts := CheckOptions{ReadData: true, CheckUnused: !unsafeNoSpaceRecovery}
|
||||
testPrune(t, opts, checkOpts)
|
||||
})
|
||||
|
||||
|
@ -84,7 +88,9 @@ func testRunForgetJSON(t testing.TB, gopts GlobalOptions, args ...string) {
|
|||
pruneOpts := PruneOptions{
|
||||
MaxUnused: "5%",
|
||||
}
|
||||
return runForget(context.TODO(), opts, pruneOpts, gopts, args)
|
||||
return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
|
||||
return runForget(context.TODO(), opts, pruneOpts, gopts, term, args)
|
||||
})
|
||||
})
|
||||
rtest.OK(t, err)
|
||||
|
||||
|
@ -138,7 +144,9 @@ func TestPruneWithDamagedRepository(t *testing.T) {
|
|||
env.gopts.backendTestHook = oldHook
|
||||
}()
|
||||
// prune should fail
|
||||
rtest.Assert(t, runPrune(context.TODO(), pruneDefaultOptions, env.gopts) == errorPacksMissing,
|
||||
rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
|
||||
return runPrune(context.TODO(), pruneDefaultOptions, env.gopts, term)
|
||||
}) == repository.ErrPacksMissing,
|
||||
"prune should have reported index not complete error")
|
||||
}
|
||||
|
||||
|
@ -218,7 +226,9 @@ func testEdgeCaseRepo(t *testing.T, tarfile string, optionsCheck CheckOptions, o
|
|||
testRunPrune(t, env.gopts, optionsPrune)
|
||||
testRunCheck(t, env.gopts)
|
||||
} else {
|
||||
rtest.Assert(t, runPrune(context.TODO(), optionsPrune, env.gopts) != nil,
|
||||
rtest.Assert(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
|
||||
return runPrune(context.TODO(), optionsPrune, env.gopts, term)
|
||||
}) != nil,
|
||||
"prune should have reported an error")
|
||||
}
|
||||
}
|
||||
|
|
|
@ -61,16 +61,22 @@ func runRecover(ctx context.Context, gopts GlobalOptions) error {
|
|||
// tree. If it is not referenced, we have a root tree.
|
||||
trees := make(map[restic.ID]bool)
|
||||
|
||||
repo.Index().Each(ctx, func(blob restic.PackedBlob) {
|
||||
err = repo.Index().Each(ctx, func(blob restic.PackedBlob) {
|
||||
if blob.Type == restic.TreeBlob {
|
||||
trees[blob.Blob.ID] = false
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
Verbosef("load %d trees\n", len(trees))
|
||||
bar = newProgressMax(!gopts.Quiet, uint64(len(trees)), "trees loaded")
|
||||
for id := range trees {
|
||||
tree, err := restic.LoadTree(ctx, repo, id)
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if err != nil {
|
||||
Warnf("unable to load tree %v: %v\n", id.Str(), err)
|
||||
continue
|
||||
|
|
|
@ -3,10 +3,8 @@ package main
|
|||
import (
|
||||
"context"
|
||||
|
||||
"github.com/restic/restic/internal/index"
|
||||
"github.com/restic/restic/internal/pack"
|
||||
"github.com/restic/restic/internal/repository"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui/termstatus"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
@ -25,7 +23,9 @@ Exit status is 0 if the command was successful, and non-zero if there was any er
|
|||
`,
|
||||
DisableAutoGenTag: true,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
return runRebuildIndex(cmd.Context(), repairIndexOptions, globalOptions)
|
||||
term, cancel := setupTermstatus()
|
||||
defer cancel()
|
||||
return runRebuildIndex(cmd.Context(), repairIndexOptions, globalOptions, term)
|
||||
},
|
||||
}
|
||||
|
||||
|
@ -55,105 +55,22 @@ func init() {
|
|||
}
|
||||
}
|
||||
|
||||
func runRebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions) error {
|
||||
func runRebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions, term *termstatus.Terminal) error {
|
||||
ctx, repo, unlock, err := openWithExclusiveLock(ctx, gopts, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
return rebuildIndex(ctx, opts, gopts, repo)
|
||||
}
|
||||
printer := newTerminalProgressPrinter(gopts.verbosity, term)
|
||||
|
||||
func rebuildIndex(ctx context.Context, opts RepairIndexOptions, gopts GlobalOptions, repo *repository.Repository) error {
|
||||
var obsoleteIndexes restic.IDs
|
||||
packSizeFromList := make(map[restic.ID]int64)
|
||||
packSizeFromIndex := make(map[restic.ID]int64)
|
||||
removePacks := restic.NewIDSet()
|
||||
|
||||
if opts.ReadAllPacks {
|
||||
// get list of old index files but start with empty index
|
||||
err := repo.List(ctx, restic.IndexFile, func(id restic.ID, _ int64) error {
|
||||
obsoleteIndexes = append(obsoleteIndexes, id)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
Verbosef("loading indexes...\n")
|
||||
mi := index.NewMasterIndex()
|
||||
err := index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, _ bool, err error) error {
|
||||
if err != nil {
|
||||
Warnf("removing invalid index %v: %v\n", id, err)
|
||||
obsoleteIndexes = append(obsoleteIndexes, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
mi.Insert(idx)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = mi.MergeFinalIndexes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = repo.SetIndex(mi)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
packSizeFromIndex = pack.Size(ctx, repo.Index(), false)
|
||||
}
|
||||
|
||||
Verbosef("getting pack files to read...\n")
|
||||
err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
|
||||
size, ok := packSizeFromIndex[id]
|
||||
if !ok || size != packSize {
|
||||
// Pack was not referenced in index or size does not match
|
||||
packSizeFromList[id] = packSize
|
||||
removePacks.Insert(id)
|
||||
}
|
||||
if !ok {
|
||||
Warnf("adding pack file to index %v\n", id)
|
||||
} else if size != packSize {
|
||||
Warnf("reindexing pack file %v with unexpected size %v instead of %v\n", id, packSize, size)
|
||||
}
|
||||
delete(packSizeFromIndex, id)
|
||||
return nil
|
||||
})
|
||||
err = repository.RepairIndex(ctx, repo, repository.RepairIndexOptions{
|
||||
ReadAllPacks: opts.ReadAllPacks,
|
||||
}, printer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for id := range packSizeFromIndex {
|
||||
// forget pack files that are referenced in the index but do not exist
|
||||
// when rebuilding the index
|
||||
removePacks.Insert(id)
|
||||
Warnf("removing not found pack file %v\n", id)
|
||||
}
|
||||
|
||||
if len(packSizeFromList) > 0 {
|
||||
Verbosef("reading pack files\n")
|
||||
bar := newProgressMax(!gopts.Quiet, uint64(len(packSizeFromList)), "packs")
|
||||
invalidFiles, err := repo.CreateIndexFromPacks(ctx, packSizeFromList, bar)
|
||||
bar.Done()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, id := range invalidFiles {
|
||||
Verboseff("skipped incomplete pack file: %v\n", id)
|
||||
}
|
||||
}
|
||||
|
||||
err = rebuildIndexFiles(ctx, gopts, repo, removePacks, obsoleteIndexes, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
Verbosef("done\n")
|
||||
|
||||
printer.P("done\n")
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -13,12 +13,15 @@ import (
|
|||
"github.com/restic/restic/internal/index"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
"github.com/restic/restic/internal/ui/termstatus"
|
||||
)
|
||||
|
||||
func testRunRebuildIndex(t testing.TB, gopts GlobalOptions) {
|
||||
rtest.OK(t, withRestoreGlobalOptions(func() error {
|
||||
globalOptions.stdout = io.Discard
|
||||
return runRebuildIndex(context.TODO(), RepairIndexOptions{}, gopts)
|
||||
return withTermStatus(gopts, func(ctx context.Context, term *termstatus.Terminal) error {
|
||||
globalOptions.stdout = io.Discard
|
||||
return runRebuildIndex(context.TODO(), RepairIndexOptions{}, gopts, term)
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
|
@ -126,12 +129,13 @@ func TestRebuildIndexFailsOnAppendOnly(t *testing.T) {
|
|||
rtest.SetupTarTestFixture(t, env.base, datafile)
|
||||
|
||||
err := withRestoreGlobalOptions(func() error {
|
||||
globalOptions.stdout = io.Discard
|
||||
|
||||
env.gopts.backendTestHook = func(r backend.Backend) (backend.Backend, error) {
|
||||
return &appendOnlyBackend{r}, nil
|
||||
}
|
||||
return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts)
|
||||
return withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
|
||||
globalOptions.stdout = io.Discard
|
||||
return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts, term)
|
||||
})
|
||||
})
|
||||
|
||||
if err == nil {
|
||||
|
|
|
@ -58,14 +58,14 @@ func runRepairPacks(ctx context.Context, gopts GlobalOptions, term *termstatus.T
|
|||
}
|
||||
defer unlock()
|
||||
|
||||
bar := newIndexProgress(gopts.Quiet, gopts.JSON)
|
||||
printer := newTerminalProgressPrinter(gopts.verbosity, term)
|
||||
|
||||
bar := newIndexTerminalProgress(gopts.Quiet, gopts.JSON, term)
|
||||
err = repo.LoadIndex(ctx, bar)
|
||||
if err != nil {
|
||||
return errors.Fatalf("%s", err)
|
||||
}
|
||||
|
||||
printer := newTerminalProgressPrinter(gopts.verbosity, term)
|
||||
|
||||
printer.P("saving backup copies of pack files to current folder")
|
||||
for id := range ids {
|
||||
f, err := os.OpenFile("pack-"+id.String(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o666)
|
||||
|
@ -82,6 +82,10 @@ func runRepairPacks(ctx context.Context, gopts GlobalOptions, term *termstatus.T
|
|||
return err
|
||||
})
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
|
@ -145,6 +145,9 @@ func runRepairSnapshots(ctx context.Context, gopts GlobalOptions, opts RepairOpt
|
|||
changedCount++
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
Verbosef("\n")
|
||||
if changedCount == 0 {
|
||||
|
|
|
@ -294,6 +294,9 @@ func runRewrite(ctx context.Context, opts RewriteOptions, gopts GlobalOptions, a
|
|||
changedCount++
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
Verbosef("\n")
|
||||
if changedCount == 0 {
|
||||
|
|
|
@ -69,6 +69,9 @@ func runSnapshots(ctx context.Context, opts SnapshotOptions, gopts GlobalOptions
|
|||
for sn := range FindFilteredSnapshots(ctx, repo, repo, &opts.SnapshotFilter, args) {
|
||||
snapshots = append(snapshots, sn)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
snapshotGroups, grouped, err := restic.GroupSnapshots(snapshots, opts.GroupBy)
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
|
@ -38,7 +38,7 @@ depending on what you are trying to calculate.
|
|||
The modes are:
|
||||
|
||||
* restore-size: (default) Counts the size of the restored files.
|
||||
* files-by-contents: Counts total size of files, where a file is
|
||||
* files-by-contents: Counts total size of unique files, where a file is
|
||||
considered unique if it has unique contents.
|
||||
* raw-data: Counts the size of blobs in the repository, regardless of
|
||||
how many files reference them.
|
||||
|
@ -117,9 +117,8 @@ func runStats(ctx context.Context, opts StatsOptions, gopts GlobalOptions, args
|
|||
return fmt.Errorf("error walking snapshot: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if opts.countMode == countModeRawData {
|
||||
|
@ -352,7 +351,10 @@ func statsDebug(ctx context.Context, repo restic.Repository) error {
|
|||
Warnf("File Type: %v\n%v\n", t, hist)
|
||||
}
|
||||
|
||||
hist := statsDebugBlobs(ctx, repo)
|
||||
hist, err := statsDebugBlobs(ctx, repo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, t := range []restic.BlobType{restic.DataBlob, restic.TreeBlob} {
|
||||
Warnf("Blob Type: %v\n%v\n\n", t, hist[t])
|
||||
}
|
||||
|
@ -370,17 +372,17 @@ func statsDebugFileType(ctx context.Context, repo restic.Lister, tpe restic.File
|
|||
return hist, err
|
||||
}
|
||||
|
||||
func statsDebugBlobs(ctx context.Context, repo restic.Repository) [restic.NumBlobTypes]*sizeHistogram {
|
||||
func statsDebugBlobs(ctx context.Context, repo restic.Repository) ([restic.NumBlobTypes]*sizeHistogram, error) {
|
||||
var hist [restic.NumBlobTypes]*sizeHistogram
|
||||
for i := 0; i < len(hist); i++ {
|
||||
hist[i] = newSizeHistogram(2 * chunker.MaxSize)
|
||||
}
|
||||
|
||||
repo.Index().Each(ctx, func(pb restic.PackedBlob) {
|
||||
err := repo.Index().Each(ctx, func(pb restic.PackedBlob) {
|
||||
hist[pb.Type].Add(uint64(pb.Length))
|
||||
})
|
||||
|
||||
return hist
|
||||
return hist, err
|
||||
}
|
||||
|
||||
type sizeClass struct {
|
||||
|
|
|
@ -122,6 +122,9 @@ func runTag(ctx context.Context, opts TagOptions, gopts GlobalOptions, args []st
|
|||
changeCnt++
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if changeCnt == 0 {
|
||||
Verbosef("no snapshots were modified\n")
|
||||
} else {
|
||||
|
|
|
@ -1,41 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
// DeleteFiles deletes the given fileList of fileType in parallel
|
||||
// it will print a warning if there is an error, but continue deleting the remaining files
|
||||
func DeleteFiles(ctx context.Context, gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) {
|
||||
_ = deleteFiles(ctx, gopts, true, repo, fileList, fileType)
|
||||
}
|
||||
|
||||
// DeleteFilesChecked deletes the given fileList of fileType in parallel
|
||||
// if an error occurs, it will cancel and return this error
|
||||
func DeleteFilesChecked(ctx context.Context, gopts GlobalOptions, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
|
||||
return deleteFiles(ctx, gopts, false, repo, fileList, fileType)
|
||||
}
|
||||
|
||||
// deleteFiles deletes the given fileList of fileType in parallel
|
||||
// if ignoreError=true, it will print a warning if there was an error, else it will abort.
|
||||
func deleteFiles(ctx context.Context, gopts GlobalOptions, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType) error {
|
||||
bar := newProgressMax(!gopts.JSON && !gopts.Quiet, 0, "files deleted")
|
||||
defer bar.Done()
|
||||
|
||||
return restic.ParallelRemove(ctx, repo, fileList, fileType, func(id restic.ID, err error) error {
|
||||
if err != nil {
|
||||
if !gopts.JSON {
|
||||
Warnf("unable to remove %v/%v from the repository\n", fileType, id)
|
||||
}
|
||||
if !ignoreError {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !gopts.JSON && gopts.verbosity > 2 {
|
||||
Verbosef("removed %v/%v\n", fileType, id)
|
||||
}
|
||||
return nil
|
||||
}, bar)
|
||||
}
|
|
@ -2,6 +2,7 @@ package main
|
|||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/spf13/pflag"
|
||||
|
@ -14,17 +15,27 @@ func initMultiSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter,
|
|||
if !addHostShorthand {
|
||||
hostShorthand = ""
|
||||
}
|
||||
flags.StringArrayVarP(&filt.Hosts, "host", hostShorthand, nil, "only consider snapshots for this `host` (can be specified multiple times)")
|
||||
flags.StringArrayVarP(&filt.Hosts, "host", hostShorthand, nil, "only consider snapshots for this `host` (can be specified multiple times) (default: $RESTIC_HOST)")
|
||||
flags.Var(&filt.Tags, "tag", "only consider snapshots including `tag[,tag,...]` (can be specified multiple times)")
|
||||
flags.StringArrayVar(&filt.Paths, "path", nil, "only consider snapshots including this (absolute) `path` (can be specified multiple times)")
|
||||
|
||||
// set default based on env if set
|
||||
if host := os.Getenv("RESTIC_HOST"); host != "" {
|
||||
filt.Hosts = []string{host}
|
||||
}
|
||||
}
|
||||
|
||||
// initSingleSnapshotFilter is used for commands that work on a single snapshot
|
||||
// MUST be combined with restic.FindFilteredSnapshot
|
||||
func initSingleSnapshotFilter(flags *pflag.FlagSet, filt *restic.SnapshotFilter) {
|
||||
flags.StringArrayVarP(&filt.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times)")
|
||||
flags.StringArrayVarP(&filt.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when snapshot ID \"latest\" is given (can be specified multiple times) (default: $RESTIC_HOST)")
|
||||
flags.Var(&filt.Tags, "tag", "only consider snapshots including `tag[,tag,...]`, when snapshot ID \"latest\" is given (can be specified multiple times)")
|
||||
flags.StringArrayVar(&filt.Paths, "path", nil, "only consider snapshots including this (absolute) `path`, when snapshot ID \"latest\" is given (can be specified multiple times)")
|
||||
|
||||
// set default based on env if set
|
||||
if host := os.Getenv("RESTIC_HOST"); host != "" {
|
||||
filt.Hosts = []string{host}
|
||||
}
|
||||
}
|
||||
|
||||
// FindFilteredSnapshots yields Snapshots, either given explicitly by `snapshotIDs` or filtered from the list of all snapshots.
|
||||
|
|
|
@ -0,0 +1,61 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/restic"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
func TestSnapshotFilter(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
args []string
|
||||
expected []string
|
||||
env string
|
||||
}{
|
||||
{
|
||||
"no value",
|
||||
[]string{},
|
||||
nil,
|
||||
"",
|
||||
},
|
||||
{
|
||||
"args only",
|
||||
[]string{"--host", "abc"},
|
||||
[]string{"abc"},
|
||||
"",
|
||||
},
|
||||
{
|
||||
"env default",
|
||||
[]string{},
|
||||
[]string{"def"},
|
||||
"def",
|
||||
},
|
||||
{
|
||||
"both",
|
||||
[]string{"--host", "abc"},
|
||||
[]string{"abc"},
|
||||
"def",
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
t.Setenv("RESTIC_HOST", test.env)
|
||||
|
||||
for _, mode := range []bool{false, true} {
|
||||
set := pflag.NewFlagSet("test", pflag.PanicOnError)
|
||||
flt := &restic.SnapshotFilter{}
|
||||
if mode {
|
||||
initMultiSnapshotFilter(set, flt, false)
|
||||
} else {
|
||||
initSingleSnapshotFilter(set, flt)
|
||||
}
|
||||
err := set.Parse(test.args)
|
||||
rtest.OK(t, err)
|
||||
|
||||
rtest.Equals(t, test.expected, flt.Hosts, "unexpected hosts")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
|
@ -26,6 +26,7 @@ import (
|
|||
"github.com/restic/restic/internal/backend/s3"
|
||||
"github.com/restic/restic/internal/backend/sema"
|
||||
"github.com/restic/restic/internal/backend/sftp"
|
||||
"github.com/restic/restic/internal/backend/smb"
|
||||
"github.com/restic/restic/internal/backend/swift"
|
||||
"github.com/restic/restic/internal/cache"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
|
@ -43,7 +44,7 @@ import (
|
|||
"golang.org/x/term"
|
||||
)
|
||||
|
||||
var version = "0.16.4-dev (compiled manually)"
|
||||
const version = "0.16.4-dev (compiled manually)"
|
||||
|
||||
// TimeFormat is the format used for all timestamps printed by restic.
|
||||
const TimeFormat = "2006-01-02 15:04:05"
|
||||
|
@ -96,9 +97,6 @@ var globalOptions = GlobalOptions{
|
|||
stderr: os.Stderr,
|
||||
}
|
||||
|
||||
var isReadingPassword bool
|
||||
var internalGlobalCtx context.Context
|
||||
|
||||
func init() {
|
||||
backends := location.NewRegistry()
|
||||
backends.Register(azure.NewFactory())
|
||||
|
@ -110,17 +108,9 @@ func init() {
|
|||
backends.Register(s3.NewFactory())
|
||||
backends.Register(sftp.NewFactory())
|
||||
backends.Register(swift.NewFactory())
|
||||
backends.Register(smb.NewFactory())
|
||||
globalOptions.backends = backends
|
||||
|
||||
var cancel context.CancelFunc
|
||||
internalGlobalCtx, cancel = context.WithCancel(context.Background())
|
||||
AddCleanupHandler(func(code int) (int, error) {
|
||||
// Must be called before the unlock cleanup handler to ensure that the latter is
|
||||
// not blocked due to limited number of backend connections, see #1434
|
||||
cancel()
|
||||
return code, nil
|
||||
})
|
||||
|
||||
f := cmdRoot.PersistentFlags()
|
||||
f.StringVarP(&globalOptions.Repo, "repo", "r", "", "`repository` to backup to or restore from (default: $RESTIC_REPOSITORY)")
|
||||
f.StringVarP(&globalOptions.RepositoryFile, "repository-file", "", "", "`file` to read the repository location from (default: $RESTIC_REPOSITORY_FILE)")
|
||||
|
@ -165,8 +155,6 @@ func init() {
|
|||
// parse target pack size from env, on error the default value will be used
|
||||
targetPackSize, _ := strconv.ParseUint(os.Getenv("RESTIC_PACK_SIZE"), 10, 32)
|
||||
globalOptions.PackSize = uint(targetPackSize)
|
||||
|
||||
restoreTerminal()
|
||||
}
|
||||
|
||||
func stdinIsTerminal() bool {
|
||||
|
@ -191,40 +179,6 @@ func stdoutTerminalWidth() int {
|
|||
return w
|
||||
}
|
||||
|
||||
// restoreTerminal installs a cleanup handler that restores the previous
|
||||
// terminal state on exit. This handler is only intended to restore the
|
||||
// terminal configuration if restic exits after receiving a signal. A regular
|
||||
// program execution must revert changes to the terminal configuration itself.
|
||||
// The terminal configuration is only restored while reading a password.
|
||||
func restoreTerminal() {
|
||||
if !term.IsTerminal(int(os.Stdout.Fd())) {
|
||||
return
|
||||
}
|
||||
|
||||
fd := int(os.Stdout.Fd())
|
||||
state, err := term.GetState(fd)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "unable to get terminal state: %v\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
AddCleanupHandler(func(code int) (int, error) {
|
||||
// Restoring the terminal configuration while restic runs in the
|
||||
// background, causes restic to get stopped on unix systems with
|
||||
// a SIGTTOU signal. Thus only restore the terminal settings if
|
||||
// they might have been modified, which is the case while reading
|
||||
// a password.
|
||||
if !isReadingPassword {
|
||||
return code, nil
|
||||
}
|
||||
err := term.Restore(fd, state)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "unable to restore terminal state: %v\n", err)
|
||||
}
|
||||
return code, err
|
||||
})
|
||||
}
|
||||
|
||||
// ClearLine creates a platform dependent string to clear the current
|
||||
// line, so it can be overwritten.
|
||||
//
|
||||
|
@ -333,24 +287,48 @@ func readPassword(in io.Reader) (password string, err error) {
|
|||
|
||||
// readPasswordTerminal reads the password from the given reader which must be a
|
||||
// tty. Prompt is printed on the writer out before attempting to read the
|
||||
// password.
|
||||
func readPasswordTerminal(in *os.File, out io.Writer, prompt string) (password string, err error) {
|
||||
fmt.Fprint(out, prompt)
|
||||
isReadingPassword = true
|
||||
buf, err := term.ReadPassword(int(in.Fd()))
|
||||
isReadingPassword = false
|
||||
fmt.Fprintln(out)
|
||||
// password. If the context is canceled, the function leaks the password reading
|
||||
// goroutine.
|
||||
func readPasswordTerminal(ctx context.Context, in *os.File, out *os.File, prompt string) (password string, err error) {
|
||||
fd := int(out.Fd())
|
||||
state, err := term.GetState(fd)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "unable to get terminal state: %v\n", err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
done := make(chan struct{})
|
||||
var buf []byte
|
||||
|
||||
go func() {
|
||||
defer close(done)
|
||||
fmt.Fprint(out, prompt)
|
||||
buf, err = term.ReadPassword(int(in.Fd()))
|
||||
fmt.Fprintln(out)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
err := term.Restore(fd, state)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "unable to restore terminal state: %v\n", err)
|
||||
}
|
||||
return "", ctx.Err()
|
||||
case <-done:
|
||||
// clean shutdown, nothing to do
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "ReadPassword")
|
||||
}
|
||||
|
||||
password = string(buf)
|
||||
return password, nil
|
||||
return string(buf), nil
|
||||
}
|
||||
|
||||
// ReadPassword reads the password from a password file, the environment
|
||||
// variable RESTIC_PASSWORD or prompts the user.
|
||||
func ReadPassword(opts GlobalOptions, prompt string) (string, error) {
|
||||
// variable RESTIC_PASSWORD or prompts the user. If the context is canceled,
|
||||
// the function leaks the password reading goroutine.
|
||||
func ReadPassword(ctx context.Context, opts GlobalOptions, prompt string) (string, error) {
|
||||
if opts.password != "" {
|
||||
return opts.password, nil
|
||||
}
|
||||
|
@ -361,7 +339,7 @@ func ReadPassword(opts GlobalOptions, prompt string) (string, error) {
|
|||
)
|
||||
|
||||
if stdinIsTerminal() {
|
||||
password, err = readPasswordTerminal(os.Stdin, os.Stderr, prompt)
|
||||
password, err = readPasswordTerminal(ctx, os.Stdin, os.Stderr, prompt)
|
||||
} else {
|
||||
password, err = readPassword(os.Stdin)
|
||||
Verbosef("reading repository password from stdin\n")
|
||||
|
@ -379,14 +357,15 @@ func ReadPassword(opts GlobalOptions, prompt string) (string, error) {
|
|||
}
|
||||
|
||||
// ReadPasswordTwice calls ReadPassword two times and returns an error when the
|
||||
// passwords don't match.
|
||||
func ReadPasswordTwice(gopts GlobalOptions, prompt1, prompt2 string) (string, error) {
|
||||
pw1, err := ReadPassword(gopts, prompt1)
|
||||
// passwords don't match. If the context is canceled, the function leaks the
|
||||
// password reading goroutine.
|
||||
func ReadPasswordTwice(ctx context.Context, gopts GlobalOptions, prompt1, prompt2 string) (string, error) {
|
||||
pw1, err := ReadPassword(ctx, gopts, prompt1)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if stdinIsTerminal() {
|
||||
pw2, err := ReadPassword(gopts, prompt2)
|
||||
pw2, err := ReadPassword(ctx, gopts, prompt2)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
@ -469,7 +448,10 @@ func OpenRepository(ctx context.Context, opts GlobalOptions) (*repository.Reposi
|
|||
}
|
||||
|
||||
for ; passwordTriesLeft > 0; passwordTriesLeft-- {
|
||||
opts.password, err = ReadPassword(opts, "enter password for repository: ")
|
||||
opts.password, err = ReadPassword(ctx, opts, "enter password for repository: ")
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
if err != nil && passwordTriesLeft > 1 {
|
||||
opts.password = ""
|
||||
fmt.Printf("%s. Try again\n", err)
|
||||
|
@ -570,16 +552,13 @@ func parseConfig(loc location.Location, opts options.Options) (interface{}, erro
|
|||
return cfg, nil
|
||||
}
|
||||
|
||||
// Open the backend specified by a location config.
|
||||
func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (backend.Backend, error) {
|
||||
func innerOpen(ctx context.Context, s string, gopts GlobalOptions, opts options.Options, create bool) (backend.Backend, error) {
|
||||
debug.Log("parsing location %v", location.StripPassword(gopts.backends, s))
|
||||
loc, err := location.Parse(gopts.backends, s)
|
||||
if err != nil {
|
||||
return nil, errors.Fatalf("parsing repository location failed: %v", err)
|
||||
}
|
||||
|
||||
var be backend.Backend
|
||||
|
||||
cfg, err := parseConfig(loc, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
@ -599,7 +578,13 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio
|
|||
return nil, errors.Fatalf("invalid backend: %q", loc.Scheme)
|
||||
}
|
||||
|
||||
be, err = factory.Open(ctx, cfg, rt, lim)
|
||||
var be backend.Backend
|
||||
if create {
|
||||
be, err = factory.Create(ctx, cfg, rt, lim)
|
||||
} else {
|
||||
be, err = factory.Open(ctx, cfg, rt, lim)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.Fatalf("unable to open repository at %v: %v", location.StripPassword(gopts.backends, s), err)
|
||||
}
|
||||
|
@ -615,6 +600,17 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio
|
|||
}
|
||||
}
|
||||
|
||||
return be, nil
|
||||
}
|
||||
|
||||
// Open the backend specified by a location config.
|
||||
func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (backend.Backend, error) {
|
||||
|
||||
be, err := innerOpen(ctx, s, gopts, opts, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// check if config is there
|
||||
fi, err := be.Stat(ctx, backend.Handle{Type: restic.ConfigFile})
|
||||
if err != nil {
|
||||
|
@ -630,31 +626,5 @@ func open(ctx context.Context, s string, gopts GlobalOptions, opts options.Optio
|
|||
|
||||
// Create the backend specified by URI.
|
||||
func create(ctx context.Context, s string, gopts GlobalOptions, opts options.Options) (backend.Backend, error) {
|
||||
debug.Log("parsing location %v", location.StripPassword(gopts.backends, s))
|
||||
loc, err := location.Parse(gopts.backends, s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg, err := parseConfig(loc, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rt, err := backend.Transport(globalOptions.TransportOptions)
|
||||
if err != nil {
|
||||
return nil, errors.Fatal(err.Error())
|
||||
}
|
||||
|
||||
factory := gopts.backends.Lookup(loc.Scheme)
|
||||
if factory == nil {
|
||||
return nil, errors.Fatalf("invalid backend: %q", loc.Scheme)
|
||||
}
|
||||
|
||||
be, err := factory.Create(ctx, cfg, rt, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return logger.New(sema.NewBackend(be)), nil
|
||||
return innerOpen(ctx, s, gopts, opts, true)
|
||||
}
|
||||
|
|
|
@ -15,23 +15,28 @@ import (
|
|||
"github.com/pkg/profile"
|
||||
)
|
||||
|
||||
var (
|
||||
listenProfile string
|
||||
memProfilePath string
|
||||
cpuProfilePath string
|
||||
traceProfilePath string
|
||||
blockProfilePath string
|
||||
insecure bool
|
||||
)
|
||||
type ProfileOptions struct {
|
||||
listen string
|
||||
memPath string
|
||||
cpuPath string
|
||||
tracePath string
|
||||
blockPath string
|
||||
insecure bool
|
||||
}
|
||||
|
||||
var profileOpts ProfileOptions
|
||||
var prof interface {
|
||||
Stop()
|
||||
}
|
||||
|
||||
func init() {
|
||||
f := cmdRoot.PersistentFlags()
|
||||
f.StringVar(&listenProfile, "listen-profile", "", "listen on this `address:port` for memory profiling")
|
||||
f.StringVar(&memProfilePath, "mem-profile", "", "write memory profile to `dir`")
|
||||
f.StringVar(&cpuProfilePath, "cpu-profile", "", "write cpu profile to `dir`")
|
||||
f.StringVar(&traceProfilePath, "trace-profile", "", "write trace to `dir`")
|
||||
f.StringVar(&blockProfilePath, "block-profile", "", "write block profile to `dir`")
|
||||
f.BoolVar(&insecure, "insecure-kdf", false, "use insecure KDF settings")
|
||||
f.StringVar(&profileOpts.listen, "listen-profile", "", "listen on this `address:port` for memory profiling")
|
||||
f.StringVar(&profileOpts.memPath, "mem-profile", "", "write memory profile to `dir`")
|
||||
f.StringVar(&profileOpts.cpuPath, "cpu-profile", "", "write cpu profile to `dir`")
|
||||
f.StringVar(&profileOpts.tracePath, "trace-profile", "", "write trace to `dir`")
|
||||
f.StringVar(&profileOpts.blockPath, "block-profile", "", "write block profile to `dir`")
|
||||
f.BoolVar(&profileOpts.insecure, "insecure-kdf", false, "use insecure KDF settings")
|
||||
}
|
||||
|
||||
type fakeTestingTB struct{}
|
||||
|
@ -41,10 +46,10 @@ func (fakeTestingTB) Logf(msg string, args ...interface{}) {
|
|||
}
|
||||
|
||||
func runDebug() error {
|
||||
if listenProfile != "" {
|
||||
fmt.Fprintf(os.Stderr, "running profile HTTP server on %v\n", listenProfile)
|
||||
if profileOpts.listen != "" {
|
||||
fmt.Fprintf(os.Stderr, "running profile HTTP server on %v\n", profileOpts.listen)
|
||||
go func() {
|
||||
err := http.ListenAndServe(listenProfile, nil)
|
||||
err := http.ListenAndServe(profileOpts.listen, nil)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "profile HTTP server listen failed: %v\n", err)
|
||||
}
|
||||
|
@ -52,16 +57,16 @@ func runDebug() error {
|
|||
}
|
||||
|
||||
profilesEnabled := 0
|
||||
if memProfilePath != "" {
|
||||
if profileOpts.memPath != "" {
|
||||
profilesEnabled++
|
||||
}
|
||||
if cpuProfilePath != "" {
|
||||
if profileOpts.cpuPath != "" {
|
||||
profilesEnabled++
|
||||
}
|
||||
if traceProfilePath != "" {
|
||||
if profileOpts.tracePath != "" {
|
||||
profilesEnabled++
|
||||
}
|
||||
if blockProfilePath != "" {
|
||||
if profileOpts.blockPath != "" {
|
||||
profilesEnabled++
|
||||
}
|
||||
|
||||
|
@ -69,30 +74,25 @@ func runDebug() error {
|
|||
return errors.Fatal("only one profile (memory, CPU, trace, or block) may be activated at the same time")
|
||||
}
|
||||
|
||||
var prof interface {
|
||||
Stop()
|
||||
if profileOpts.memPath != "" {
|
||||
prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.MemProfile, profile.ProfilePath(profileOpts.memPath))
|
||||
} else if profileOpts.cpuPath != "" {
|
||||
prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.CPUProfile, profile.ProfilePath(profileOpts.cpuPath))
|
||||
} else if profileOpts.tracePath != "" {
|
||||
prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.TraceProfile, profile.ProfilePath(profileOpts.tracePath))
|
||||
} else if profileOpts.blockPath != "" {
|
||||
prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.BlockProfile, profile.ProfilePath(profileOpts.blockPath))
|
||||
}
|
||||
|
||||
if memProfilePath != "" {
|
||||
prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.MemProfile, profile.ProfilePath(memProfilePath))
|
||||
} else if cpuProfilePath != "" {
|
||||
prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.CPUProfile, profile.ProfilePath(cpuProfilePath))
|
||||
} else if traceProfilePath != "" {
|
||||
prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.TraceProfile, profile.ProfilePath(traceProfilePath))
|
||||
} else if blockProfilePath != "" {
|
||||
prof = profile.Start(profile.Quiet, profile.NoShutdownHook, profile.BlockProfile, profile.ProfilePath(blockProfilePath))
|
||||
}
|
||||
|
||||
if prof != nil {
|
||||
AddCleanupHandler(func(code int) (int, error) {
|
||||
prof.Stop()
|
||||
return code, nil
|
||||
})
|
||||
}
|
||||
|
||||
if insecure {
|
||||
if profileOpts.insecure {
|
||||
repository.TestUseLowSecurityKDFParameters(fakeTestingTB{})
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func stopDebug() {
|
||||
if prof != nil {
|
||||
prof.Stop()
|
||||
}
|
||||
}
|
||||
|
|
|
@ -5,3 +5,6 @@ package main
|
|||
|
||||
// runDebug is a noop without the debug tag.
|
||||
func runDebug() error { return nil }
|
||||
|
||||
// stopDebug is a noop without the debug tag.
|
||||
func stopDebug() {}
|
||||
|
|
|
@ -252,11 +252,11 @@ func listTreePacks(gopts GlobalOptions, t *testing.T) restic.IDSet {
|
|||
|
||||
rtest.OK(t, r.LoadIndex(ctx, nil))
|
||||
treePacks := restic.NewIDSet()
|
||||
r.Index().Each(ctx, func(pb restic.PackedBlob) {
|
||||
rtest.OK(t, r.Index().Each(ctx, func(pb restic.PackedBlob) {
|
||||
if pb.Type == restic.TreeBlob {
|
||||
treePacks.Insert(pb.PackID)
|
||||
}
|
||||
})
|
||||
}))
|
||||
|
||||
return treePacks
|
||||
}
|
||||
|
@ -280,11 +280,11 @@ func removePacksExcept(gopts GlobalOptions, t testing.TB, keep restic.IDSet, rem
|
|||
rtest.OK(t, r.LoadIndex(ctx, nil))
|
||||
|
||||
treePacks := restic.NewIDSet()
|
||||
r.Index().Each(ctx, func(pb restic.PackedBlob) {
|
||||
rtest.OK(t, r.Index().Each(ctx, func(pb restic.PackedBlob) {
|
||||
if pb.Type == restic.TreeBlob {
|
||||
treePacks.Insert(pb.PackID)
|
||||
}
|
||||
})
|
||||
}))
|
||||
|
||||
// remove all packs containing data blobs
|
||||
rtest.OK(t, r.List(ctx, restic.PackFile, func(id restic.ID, size int64) error {
|
||||
|
|
|
@ -12,6 +12,7 @@ import (
|
|||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
"github.com/restic/restic/internal/ui/termstatus"
|
||||
)
|
||||
|
||||
func TestCheckRestoreNoLock(t *testing.T) {
|
||||
|
@ -88,8 +89,12 @@ func TestListOnce(t *testing.T) {
|
|||
testRunPrune(t, env.gopts, pruneOpts)
|
||||
rtest.OK(t, runCheck(context.TODO(), checkOpts, env.gopts, nil))
|
||||
|
||||
rtest.OK(t, runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts))
|
||||
rtest.OK(t, runRebuildIndex(context.TODO(), RepairIndexOptions{ReadAllPacks: true}, env.gopts))
|
||||
rtest.OK(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
|
||||
return runRebuildIndex(context.TODO(), RepairIndexOptions{}, env.gopts, term)
|
||||
}))
|
||||
rtest.OK(t, withTermStatus(env.gopts, func(ctx context.Context, term *termstatus.Terminal) error {
|
||||
return runRebuildIndex(context.TODO(), RepairIndexOptions{ReadAllPacks: true}, env.gopts, term)
|
||||
}))
|
||||
}
|
||||
|
||||
type writeToOnly struct {
|
||||
|
|
|
@ -21,18 +21,11 @@ func internalOpenWithLocked(ctx context.Context, gopts GlobalOptions, dryRun boo
|
|||
Verbosef("%s", msg)
|
||||
}
|
||||
}, Warnf)
|
||||
|
||||
unlock = lock.Unlock
|
||||
// make sure that a repository is unlocked properly and after cancel() was
|
||||
// called by the cleanup handler in global.go
|
||||
AddCleanupHandler(func(code int) (int, error) {
|
||||
lock.Unlock()
|
||||
return code, nil
|
||||
})
|
||||
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
unlock = lock.Unlock
|
||||
} else {
|
||||
repo.SetDryRun()
|
||||
}
|
||||
|
|
|
@ -3,6 +3,7 @@ package main
|
|||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
|
@ -24,6 +25,8 @@ func init() {
|
|||
_, _ = maxprocs.Set()
|
||||
}
|
||||
|
||||
var ErrOK = errors.New("ok")
|
||||
|
||||
// cmdRoot is the base command when no other command has been specified.
|
||||
var cmdRoot = &cobra.Command{
|
||||
Use: "restic",
|
||||
|
@ -74,6 +77,9 @@ The full documentation can be found at https://restic.readthedocs.io/ .
|
|||
// enabled)
|
||||
return runDebug()
|
||||
},
|
||||
PersistentPostRun: func(_ *cobra.Command, _ []string) {
|
||||
stopDebug()
|
||||
},
|
||||
}
|
||||
|
||||
// Distinguish commands that need the password from those that work without,
|
||||
|
@ -88,8 +94,6 @@ func needsPassword(cmd string) bool {
|
|||
}
|
||||
}
|
||||
|
||||
var logBuffer = bytes.NewBuffer(nil)
|
||||
|
||||
func tweakGoGC() {
|
||||
// lower GOGC from 100 to 50, unless it was manually overwritten by the user
|
||||
oldValue := godebug.SetGCPercent(50)
|
||||
|
@ -102,6 +106,7 @@ func main() {
|
|||
tweakGoGC()
|
||||
// install custom global logger into a buffer, if an error occurs
|
||||
// we can show the logs
|
||||
logBuffer := bytes.NewBuffer(nil)
|
||||
log.SetOutput(logBuffer)
|
||||
|
||||
err := feature.Flag.Apply(os.Getenv("RESTIC_FEATURES"), func(s string) {
|
||||
|
@ -115,7 +120,16 @@ func main() {
|
|||
debug.Log("main %#v", os.Args)
|
||||
debug.Log("restic %s compiled with %v on %v/%v",
|
||||
version, runtime.Version(), runtime.GOOS, runtime.GOARCH)
|
||||
err = cmdRoot.ExecuteContext(internalGlobalCtx)
|
||||
|
||||
ctx := createGlobalContext()
|
||||
err = cmdRoot.ExecuteContext(ctx)
|
||||
|
||||
if err == nil {
|
||||
err = ctx.Err()
|
||||
} else if err == ErrOK {
|
||||
// ErrOK overwrites context cancelation errors
|
||||
err = nil
|
||||
}
|
||||
|
||||
switch {
|
||||
case restic.IsAlreadyLocked(err):
|
||||
|
@ -137,11 +151,13 @@ func main() {
|
|||
}
|
||||
|
||||
var exitCode int
|
||||
switch err {
|
||||
case nil:
|
||||
switch {
|
||||
case err == nil:
|
||||
exitCode = 0
|
||||
case ErrInvalidSourceData:
|
||||
case err == ErrInvalidSourceData:
|
||||
exitCode = 3
|
||||
case errors.Is(err, context.Canceled):
|
||||
exitCode = 130
|
||||
default:
|
||||
exitCode = 1
|
||||
}
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
|
@ -56,7 +57,7 @@ func initSecondaryRepoOptions(f *pflag.FlagSet, opts *secondaryRepoOptions, repo
|
|||
opts.PasswordCommand = os.Getenv("RESTIC_FROM_PASSWORD_COMMAND")
|
||||
}
|
||||
|
||||
func fillSecondaryGlobalOpts(opts secondaryRepoOptions, gopts GlobalOptions, repoPrefix string) (GlobalOptions, bool, error) {
|
||||
func fillSecondaryGlobalOpts(ctx context.Context, opts secondaryRepoOptions, gopts GlobalOptions, repoPrefix string) (GlobalOptions, bool, error) {
|
||||
if opts.Repo == "" && opts.RepositoryFile == "" && opts.LegacyRepo == "" && opts.LegacyRepositoryFile == "" {
|
||||
return GlobalOptions{}, false, errors.Fatal("Please specify a source repository location (--from-repo or --from-repository-file)")
|
||||
}
|
||||
|
@ -109,7 +110,7 @@ func fillSecondaryGlobalOpts(opts secondaryRepoOptions, gopts GlobalOptions, rep
|
|||
return GlobalOptions{}, false, err
|
||||
}
|
||||
}
|
||||
dstGopts.password, err = ReadPassword(dstGopts, "enter password for "+repoPrefix+" repository: ")
|
||||
dstGopts.password, err = ReadPassword(ctx, dstGopts, "enter password for "+repoPrefix+" repository: ")
|
||||
if err != nil {
|
||||
return GlobalOptions{}, false, err
|
||||
}
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
@ -170,7 +171,7 @@ func TestFillSecondaryGlobalOpts(t *testing.T) {
|
|||
|
||||
// Test all valid cases
|
||||
for _, testCase := range validSecondaryRepoTestCases {
|
||||
DstGOpts, isFromRepo, err := fillSecondaryGlobalOpts(testCase.Opts, gOpts, "destination")
|
||||
DstGOpts, isFromRepo, err := fillSecondaryGlobalOpts(context.TODO(), testCase.Opts, gOpts, "destination")
|
||||
rtest.OK(t, err)
|
||||
rtest.Equals(t, DstGOpts, testCase.DstGOpts)
|
||||
rtest.Equals(t, isFromRepo, testCase.FromRepo)
|
||||
|
@ -178,7 +179,7 @@ func TestFillSecondaryGlobalOpts(t *testing.T) {
|
|||
|
||||
// Test all invalid cases
|
||||
for _, testCase := range invalidSecondaryRepoTestCases {
|
||||
_, _, err := fillSecondaryGlobalOpts(testCase.Opts, gOpts, "destination")
|
||||
_, _, err := fillSecondaryGlobalOpts(context.TODO(), testCase.Opts, gOpts, "destination")
|
||||
rtest.Assert(t, err != nil, "Expected error, but function did not return an error")
|
||||
}
|
||||
}
|
||||
|
|
|
@ -652,6 +652,34 @@ The region, where a bucket should be created, can be specified with the ``-o gs.
|
|||
|
||||
.. _other-services:
|
||||
|
||||
SMB/CIFS
|
||||
********
|
||||
|
||||
In order to backup data to SMB/CIFS, you must specify the host (with port if not default port `445`) as the backend.
|
||||
You must first setup the following environment variables with the SMB credentials and the domain if it is not the default `WORKGROUP`.
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ export RESTIC_SMB_USER=<MY_SMB_USER>
|
||||
$ export RESTIC_SMB_PASSWORD=<MY_SMB_PASSWORD>
|
||||
$ export RESTIC_SMB_DOMAIN=<MY_SMB_DOMAIN>
|
||||
|
||||
|
||||
Once the server is configured, the setup of the SFTP repository can
|
||||
simply be achieved by changing the URL scheme in the ``init`` command:
|
||||
|
||||
.. code-block:: console
|
||||
|
||||
$ restic -r smb://user@host:445/sharename/restic-repo init
|
||||
enter password for new repository:
|
||||
enter password again:
|
||||
created restic repository c7s8ffs329 at smb://host:445/sharename/restic-repo
|
||||
Please note that knowledge of your password is required to access the repository.
|
||||
Losing your password means that your data is irrecoverably lost.
|
||||
|
||||
Optionally, you can also pass the ``user``, ``password`` and ``domain`` as options. Configurations specified as options take highest precendence.
|
||||
You can also specify other smb specific optional configurations like ``dialect``, ``client-guid``, ``require-message-signing``, ``idle-timeout`` and ``connections`` as options.
|
||||
|
||||
Other Services via rclone
|
||||
*************************
|
||||
|
||||
|
|
|
@ -664,6 +664,10 @@ environment variables. The following lists these environment variables:
|
|||
OS_STORAGE_URL Storage URL for token authentication
|
||||
OS_AUTH_TOKEN Auth token for token authentication
|
||||
|
||||
RESTIC_SMB_USER SMB user for NTLM authentication
|
||||
RESTIC_SMB_PASSWORD SMB password for NTLM authentication
|
||||
RESTIC_SMB_DOMAIN DOMAIN for SMB authentication
|
||||
|
||||
RCLONE_BWLIMIT rclone bandwidth limit
|
||||
|
||||
RESTIC_REST_USERNAME Restic REST Server username
|
||||
|
|
4
go.mod
4
go.mod
|
@ -13,8 +13,9 @@ require (
|
|||
github.com/go-ole/go-ole v1.3.0
|
||||
github.com/google/go-cmp v0.6.0
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7
|
||||
github.com/hirochachacha/go-smb2 v1.1.0
|
||||
github.com/klauspost/compress v1.17.7
|
||||
github.com/minio/minio-go/v7 v7.0.66
|
||||
github.com/minio/minio-go/v7 v7.0.66
|
||||
github.com/minio/sha256-simd v1.0.1
|
||||
github.com/ncw/swift/v2 v2.0.2
|
||||
github.com/peterbourgon/unixtransport v0.0.4
|
||||
|
@ -48,6 +49,7 @@ require (
|
|||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/felixge/fgprof v0.9.3 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/geoffgarside/ber v1.1.0 // indirect
|
||||
github.com/go-logr/logr v1.4.1 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.2.0 // indirect
|
||||
|
|
5
go.sum
5
go.sum
|
@ -57,6 +57,8 @@ github.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=
|
|||
github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/geoffgarside/ber v1.1.0 h1:qTmFG4jJbwiSzSXoNJeHcOprVzZ8Ulde2Rrrifu5U9w=
|
||||
github.com/geoffgarside/ber v1.1.0/go.mod h1:jVPKeCbj6MvQZhwLYsGwaGI52oUorHoHKNecGT85ZCc=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
|
||||
github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
|
@ -109,6 +111,8 @@ github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/
|
|||
github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/hirochachacha/go-smb2 v1.1.0 h1:b6hs9qKIql9eVXAiN0M2wSFY5xnhbHAQoCwRKbaRTZI=
|
||||
github.com/hirochachacha/go-smb2 v1.1.0/go.mod h1:8F1A4d5EZzrGu5R7PU163UcMRDJQl4FtcxjBfsY8TZE=
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
|
||||
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
|
@ -204,6 +208,7 @@ go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnw
|
|||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
|
||||
golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
|
||||
|
|
|
@ -303,7 +303,7 @@ func generateFiles() {
|
|||
}
|
||||
}
|
||||
|
||||
var versionPattern = `var version = ".*"`
|
||||
var versionPattern = `const version = ".*"`
|
||||
|
||||
const versionCodeFile = "cmd/restic/global.go"
|
||||
|
||||
|
@ -313,7 +313,7 @@ func updateVersion() {
|
|||
die("unable to write version to file: %v", err)
|
||||
}
|
||||
|
||||
newVersion := fmt.Sprintf("var version = %q", opts.Version)
|
||||
newVersion := fmt.Sprintf("const version = %q", opts.Version)
|
||||
replace(versionCodeFile, versionPattern, newVersion)
|
||||
|
||||
if len(uncommittedChanges("VERSION")) > 0 || len(uncommittedChanges(versionCodeFile)) > 0 {
|
||||
|
@ -323,7 +323,7 @@ func updateVersion() {
|
|||
}
|
||||
|
||||
func updateVersionDev() {
|
||||
newVersion := fmt.Sprintf(`var version = "%s-dev (compiled manually)"`, opts.Version)
|
||||
newVersion := fmt.Sprintf(`const version = "%s-dev (compiled manually)"`, opts.Version)
|
||||
replace(versionCodeFile, versionPattern, newVersion)
|
||||
|
||||
msg("committing cmd/restic/global.go with dev version")
|
||||
|
|
|
@ -380,6 +380,7 @@ func (fn *FutureNode) take(ctx context.Context) futureNodeResult {
|
|||
return res
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return futureNodeResult{err: ctx.Err()}
|
||||
}
|
||||
return futureNodeResult{err: errors.Errorf("no result")}
|
||||
}
|
||||
|
|
|
@ -1430,7 +1430,7 @@ func TestArchiverSnapshot(t *testing.T) {
|
|||
}
|
||||
TestEnsureSnapshot(t, repo, snapshotID, want)
|
||||
|
||||
checker.TestCheckRepo(t, repo)
|
||||
checker.TestCheckRepo(t, repo, false)
|
||||
|
||||
// check that the snapshot contains the targets with absolute paths
|
||||
for i, target := range sn.Paths {
|
||||
|
@ -1590,7 +1590,7 @@ func TestArchiverSnapshotSelect(t *testing.T) {
|
|||
}
|
||||
TestEnsureSnapshot(t, repo, snapshotID, want)
|
||||
|
||||
checker.TestCheckRepo(t, repo)
|
||||
checker.TestCheckRepo(t, repo, false)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
@ -1794,7 +1794,7 @@ func TestArchiverParent(t *testing.T) {
|
|||
t.Logf("second backup saved as %v", secondSnapshotID.Str())
|
||||
t.Logf("testfs: %v", testFS)
|
||||
|
||||
checker.TestCheckRepo(t, repo)
|
||||
checker.TestCheckRepo(t, repo, false)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
@ -1927,7 +1927,7 @@ func TestArchiverErrorReporting(t *testing.T) {
|
|||
}
|
||||
TestEnsureSnapshot(t, repo, snapshotID, want)
|
||||
|
||||
checker.TestCheckRepo(t, repo)
|
||||
checker.TestCheckRepo(t, repo, false)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
@ -2288,7 +2288,7 @@ func TestMetadataChanged(t *testing.T) {
|
|||
// make sure the content matches
|
||||
TestEnsureFileContent(context.Background(), t, repo, "testfile", node3, files["testfile"].(TestFile))
|
||||
|
||||
checker.TestCheckRepo(t, repo)
|
||||
checker.TestCheckRepo(t, repo, false)
|
||||
}
|
||||
|
||||
func TestRacyFileSwap(t *testing.T) {
|
||||
|
|
|
@ -90,6 +90,10 @@ func (s *TreeSaver) save(ctx context.Context, job *saveTreeJob) (*restic.Node, I
|
|||
// return the error if it wasn't ignored
|
||||
if fnr.err != nil {
|
||||
debug.Log("err for %v: %v", fnr.snPath, fnr.err)
|
||||
if fnr.err == context.Canceled {
|
||||
return nil, stats, fnr.err
|
||||
}
|
||||
|
||||
fnr.err = s.errFn(fnr.target, fnr.err)
|
||||
if fnr.err == nil {
|
||||
// ignore error
|
||||
|
|
|
@ -4,6 +4,7 @@ import (
|
|||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"net/http"
|
||||
|
@ -41,7 +42,7 @@ func NewFactory() location.Factory {
|
|||
)
|
||||
}
|
||||
|
||||
var errNotFound = errors.New("not found")
|
||||
var errNotFound = fmt.Errorf("not found")
|
||||
|
||||
const connectionCount = 2
|
||||
|
||||
|
|
|
@ -0,0 +1,143 @@
|
|||
package smb
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/options"
|
||||
)
|
||||
|
||||
// Config contains all configuration necessary to connect to an SMB server
|
||||
type Config struct {
|
||||
Host string
|
||||
Port int
|
||||
ShareName string
|
||||
Path string
|
||||
|
||||
User string `option:"user" help:"specify the SMB user for NTLM authentication."`
|
||||
Password options.SecretString `option:"password" help:"specify the SMB password for NTLM authentication."`
|
||||
Domain string `option:"domain" help:"specify the domain for authentication."`
|
||||
|
||||
Layout string `option:"layout" help:"use this backend directory layout (default: auto-detect)"`
|
||||
Connections uint `option:"connections" help:"set a limit for the number of concurrent operations (default: 5)"`
|
||||
IdleTimeout time.Duration `option:"idle-timeout" help:"Max time in seconds before closing idle connections. If no connections have been returned to the connection pool in the time given, the connection pool will be emptied. Set to 0 to keep connections indefinitely.(default: 60)"`
|
||||
RequireMessageSigning bool `option:"require-message-signing" help:"Mandates message signing otherwise does not allow the connection. If this is false, messaging signing is just enabled and not enforced. (default: false)"`
|
||||
Dialect uint16 `option:"dialect" help:"Force a specific dialect to be used. For SMB311 use '785', for SMB302 use '770', for SMB300 use '768', for SMB210 use '528', for SMB202 use '514', for SMB2 use '767'. If unspecfied (0), following dialects are tried in order - SMB311, SMB302, SMB300, SMB210, SMB202 (default: 0)"`
|
||||
ClientGUID string `option:"client-guid" help:"A 16-byte GUID to uniquely identify a client. If not specific a random GUID is used. (default: \"\")"`
|
||||
}
|
||||
|
||||
const (
|
||||
DefaultSmbPort int = 445 // DefaultSmbPort returns the default port for SMB
|
||||
DefaultDomain string = "WORKGROUP" // DefaultDomain returns the default domain for SMB
|
||||
DefaultConnections uint = 5 // DefaultConnections returns the number of concurrent connections for SMB.
|
||||
DefaultIdleTimeout time.Duration = 60 * time.Second // DefaultIdleTimeout returns the default max time before closing idle connections for SMB.
|
||||
)
|
||||
|
||||
// NewConfig returns a new Config with the default values filled in.
|
||||
func NewConfig() Config {
|
||||
return Config{
|
||||
Port: DefaultSmbPort,
|
||||
Domain: DefaultDomain,
|
||||
IdleTimeout: DefaultIdleTimeout,
|
||||
Connections: DefaultConnections,
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
options.Register("smb", Config{})
|
||||
}
|
||||
|
||||
// ParseConfig parses the string s and extracts the s3 config. The
|
||||
// supported configuration format is smb://[user@]host[:port]/sharename/directory.
|
||||
// User and port are optional. Default port is 445.
|
||||
func ParseConfig(s string) (*Config, error) {
|
||||
var repo string
|
||||
switch {
|
||||
case strings.HasPrefix(s, "smb://"):
|
||||
repo = s
|
||||
case strings.HasPrefix(s, "smb:"):
|
||||
repo = "smb://" + s[4:]
|
||||
default:
|
||||
return nil, errors.New("smb: invalid format")
|
||||
}
|
||||
var user, host, port, dir string
|
||||
|
||||
// parse the "smb://user@host/sharename/directory." url format
|
||||
url, err := url.Parse(repo)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
if url.User != nil {
|
||||
user = url.User.Username()
|
||||
//Intentionally not allowing passwords to be set in url as
|
||||
//it can cause issues when passwords have special characters
|
||||
//like '@' and it is not recommended to pass passwords in the url.
|
||||
}
|
||||
|
||||
host = url.Hostname()
|
||||
if host == "" {
|
||||
return nil, errors.New("smb: invalid format, host name not found")
|
||||
}
|
||||
port = url.Port()
|
||||
dir = url.Path
|
||||
if dir == "" {
|
||||
return nil, errors.Errorf("smb: invalid format, sharename/directory not found")
|
||||
}
|
||||
|
||||
dir = dir[1:]
|
||||
|
||||
var portNum int
|
||||
if port == "" {
|
||||
portNum = DefaultSmbPort
|
||||
} else {
|
||||
var err error
|
||||
portNum, err = strconv.Atoi(port)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
sharename, directory, _ := strings.Cut(dir, "/")
|
||||
|
||||
return createConfig(user, host, portNum, sharename, directory)
|
||||
}
|
||||
|
||||
func createConfig(user string, host string, port int, sharename, directory string) (*Config, error) {
|
||||
if host == "" {
|
||||
return nil, errors.New("smb: invalid format, Host not found")
|
||||
}
|
||||
|
||||
if directory != "" {
|
||||
directory = path.Clean(directory)
|
||||
}
|
||||
|
||||
cfg := NewConfig()
|
||||
cfg.User = user
|
||||
cfg.Host = host
|
||||
cfg.Port = port
|
||||
cfg.ShareName = sharename
|
||||
cfg.Path = directory
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
// ApplyEnvironment saves values from the environment to the config.
|
||||
func (cfg *Config) ApplyEnvironment(prefix string) error {
|
||||
if cfg.User == "" {
|
||||
cfg.User = os.Getenv(prefix + "RESTIC_SMB_USER")
|
||||
}
|
||||
if cfg.Password.String() == "" {
|
||||
cfg.Password = options.NewSecretString(os.Getenv(prefix + "RESTIC_SMB_PASSWORD"))
|
||||
}
|
||||
if cfg.Domain == "" {
|
||||
cfg.Domain = os.Getenv(prefix + "RESTIC_SMB_DOMAIN")
|
||||
}
|
||||
if cfg.Domain == "" {
|
||||
cfg.Domain = DefaultDomain
|
||||
}
|
||||
return nil
|
||||
}
|
|
@ -0,0 +1,79 @@
|
|||
package smb
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/backend/test"
|
||||
)
|
||||
|
||||
var configTests = []test.ConfigTestData[Config]{
|
||||
{S: "smb://user@host/sharename/directory",
|
||||
Cfg: Config{
|
||||
Host: "host",
|
||||
Port: DefaultSmbPort,
|
||||
User: "user",
|
||||
Domain: DefaultDomain,
|
||||
ShareName: "sharename",
|
||||
Path: "directory",
|
||||
Connections: DefaultConnections,
|
||||
IdleTimeout: DefaultIdleTimeout,
|
||||
}},
|
||||
{S: "smb://user@host:456/sharename/directory",
|
||||
Cfg: Config{
|
||||
Host: "host",
|
||||
Port: 456,
|
||||
User: "user",
|
||||
Domain: DefaultDomain,
|
||||
ShareName: "sharename",
|
||||
Path: "directory",
|
||||
Connections: DefaultConnections,
|
||||
IdleTimeout: DefaultIdleTimeout,
|
||||
}},
|
||||
{S: "smb://host/sharename/directory",
|
||||
Cfg: Config{
|
||||
Host: "host",
|
||||
Port: DefaultSmbPort,
|
||||
Domain: DefaultDomain,
|
||||
ShareName: "sharename",
|
||||
Path: "directory",
|
||||
Connections: DefaultConnections,
|
||||
IdleTimeout: DefaultIdleTimeout,
|
||||
}},
|
||||
{S: "smb://host:446/sharename/directory",
|
||||
Cfg: Config{
|
||||
Host: "host",
|
||||
Port: 446,
|
||||
Domain: DefaultDomain,
|
||||
ShareName: "sharename",
|
||||
Path: "directory",
|
||||
Connections: DefaultConnections,
|
||||
IdleTimeout: DefaultIdleTimeout,
|
||||
}},
|
||||
{S: "smb:user@host:466/sharename/directory",
|
||||
Cfg: Config{
|
||||
Host: "host",
|
||||
Port: 466,
|
||||
User: "user",
|
||||
Domain: DefaultDomain,
|
||||
ShareName: "sharename",
|
||||
Path: "directory",
|
||||
Connections: DefaultConnections,
|
||||
IdleTimeout: DefaultIdleTimeout,
|
||||
}},
|
||||
}
|
||||
|
||||
func TestParseConfig(t *testing.T) {
|
||||
test.ParseConfigTester(t, ParseConfig, configTests)
|
||||
}
|
||||
|
||||
func TestParseError(t *testing.T) {
|
||||
const prefix = "smb: invalid format,"
|
||||
|
||||
for _, s := range []string{"", "/", "//", "host", "user@host", "user@host:445", "/sharename/directory"} {
|
||||
_, err := ParseConfig("smb://" + s)
|
||||
if err == nil || !strings.HasPrefix(err.Error(), prefix) {
|
||||
t.Errorf("expected %q, got %q", prefix, err)
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,238 @@
|
|||
package smb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/hirochachacha/go-smb2"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
)
|
||||
|
||||
// Parts of this code have been adapted from Rclone (https://github.com/rclone)
|
||||
// Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/
|
||||
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
// THE SOFTWARE.
|
||||
|
||||
// conn encapsulates a SMB client and corresponding SMB client
|
||||
type conn struct {
|
||||
conn *net.Conn
|
||||
smbSession *smb2.Session
|
||||
smbShare *smb2.Share
|
||||
shareName string
|
||||
}
|
||||
|
||||
// Closes the connection
|
||||
func (c *conn) close() (err error) {
|
||||
if c.smbShare != nil {
|
||||
err = c.smbShare.Umount()
|
||||
}
|
||||
sessionLogoffErr := c.smbSession.Logoff()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return sessionLogoffErr
|
||||
}
|
||||
|
||||
// True if it's closed
|
||||
func (c *conn) closed() bool {
|
||||
var nopErr error
|
||||
if c.smbShare != nil {
|
||||
// stat the current directory
|
||||
_, nopErr = c.smbShare.Stat(".")
|
||||
} else {
|
||||
// list the shares
|
||||
_, nopErr = c.smbSession.ListSharenames()
|
||||
}
|
||||
return nopErr == nil
|
||||
}
|
||||
|
||||
// Show that we are using a SMB session
|
||||
//
|
||||
// Call removeSession() when done
|
||||
func (b *Backend) addSession() {
|
||||
atomic.AddInt32(&b.sessions, 1)
|
||||
}
|
||||
|
||||
// Show the SMB session is no longer in use
|
||||
func (b *Backend) removeSession() {
|
||||
atomic.AddInt32(&b.sessions, -1)
|
||||
}
|
||||
|
||||
// getSessions shows whether there are any sessions in use
|
||||
func (b *Backend) getSessions() int32 {
|
||||
return atomic.LoadInt32(&b.sessions)
|
||||
}
|
||||
|
||||
// dial starts a client connection to the given SMB server. It is a
|
||||
// convenience function that connects to the given network address,
|
||||
// initiates the SMB handshake, and then returns a session for SMB communication.
|
||||
func (b *Backend) dial(ctx context.Context, network, addr string) (*conn, error) {
|
||||
dialer := net.Dialer{}
|
||||
tconn, err := dialer.Dial(network, addr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var clientID [16]byte
|
||||
if b.ClientGUID != "" {
|
||||
copy(clientID[:], []byte(b.ClientGUID))
|
||||
}
|
||||
|
||||
d := &smb2.Dialer{
|
||||
Negotiator: smb2.Negotiator{
|
||||
RequireMessageSigning: b.RequireMessageSigning,
|
||||
SpecifiedDialect: b.Dialect,
|
||||
ClientGuid: clientID,
|
||||
},
|
||||
Initiator: &smb2.NTLMInitiator{
|
||||
User: b.User,
|
||||
Password: b.Password.Unwrap(),
|
||||
Domain: b.Domain,
|
||||
},
|
||||
}
|
||||
|
||||
session, err := d.DialContext(ctx, tconn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &conn{
|
||||
smbSession: session,
|
||||
conn: &tconn,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Open a new connection to the SMB server.
|
||||
func (b *Backend) newConnection(share string) (c *conn, err error) {
|
||||
// As we are pooling these connections we need to decouple
|
||||
// them from the current context
|
||||
ctx := context.Background()
|
||||
|
||||
c, err = b.dial(ctx, "tcp", b.Host+":"+strconv.Itoa(b.Port))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("couldn't connect SMB: %w", err)
|
||||
}
|
||||
|
||||
if share != "" {
|
||||
// mount the specified share as well if user requested
|
||||
c.smbShare, err = c.smbSession.Mount(share)
|
||||
if err != nil {
|
||||
_ = c.smbSession.Logoff()
|
||||
return nil, fmt.Errorf("couldn't initialize SMB: %w", err)
|
||||
}
|
||||
c.smbShare = c.smbShare.WithContext(ctx)
|
||||
}
|
||||
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// Ensure the specified share is mounted or the session is unmounted
|
||||
func (c *conn) mountShare(share string) (err error) {
|
||||
if c.shareName == share {
|
||||
return nil
|
||||
}
|
||||
if c.smbShare != nil {
|
||||
err = c.smbShare.Umount()
|
||||
c.smbShare = nil
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if share != "" {
|
||||
c.smbShare, err = c.smbSession.Mount(share)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
c.shareName = share
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get a SMB connection from the pool, or open a new one
|
||||
func (b *Backend) getConnection(_ context.Context, share string) (c *conn, err error) {
|
||||
b.poolMu.Lock()
|
||||
for len(b.pool) > 0 {
|
||||
c = b.pool[0]
|
||||
b.pool = b.pool[1:]
|
||||
err = c.mountShare(share)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
debug.Log("Discarding unusable SMB connection: %v", err)
|
||||
c = nil
|
||||
}
|
||||
b.poolMu.Unlock()
|
||||
if c != nil {
|
||||
return c, nil
|
||||
}
|
||||
c, err = b.newConnection(share)
|
||||
return c, err
|
||||
}
|
||||
|
||||
// Return a SMB connection to the pool
|
||||
func (b *Backend) putConnection(c *conn) {
|
||||
var nopErr error
|
||||
if c.smbShare != nil {
|
||||
// stat the current directory
|
||||
_, nopErr = c.smbShare.Stat(".")
|
||||
} else {
|
||||
// list the shares
|
||||
_, nopErr = c.smbSession.ListSharenames()
|
||||
}
|
||||
if nopErr != nil {
|
||||
debug.Log("Connection failed, closing: %v", nopErr)
|
||||
_ = c.close()
|
||||
return
|
||||
}
|
||||
|
||||
b.poolMu.Lock()
|
||||
b.pool = append(b.pool, c)
|
||||
b.drain.Reset(b.Config.IdleTimeout) // nudge on the pool emptying timer
|
||||
b.poolMu.Unlock()
|
||||
}
|
||||
|
||||
// Drain the pool of any connections
|
||||
func (b *Backend) drainPool() (err error) {
|
||||
b.poolMu.Lock()
|
||||
defer b.poolMu.Unlock()
|
||||
if sessions := b.getSessions(); sessions != 0 {
|
||||
debug.Log("Not closing %d unused connections as %d sessions active", len(b.pool), sessions)
|
||||
b.drain.Reset(b.Config.IdleTimeout) // nudge on the pool emptying timer
|
||||
return nil
|
||||
}
|
||||
if b.Config.IdleTimeout > 0 {
|
||||
b.drain.Stop()
|
||||
}
|
||||
if len(b.pool) != 0 {
|
||||
debug.Log("Closing %d unused connections", len(b.pool))
|
||||
}
|
||||
for i, c := range b.pool {
|
||||
if !c.closed() {
|
||||
cErr := c.close()
|
||||
if cErr != nil {
|
||||
err = cErr
|
||||
}
|
||||
}
|
||||
b.pool[i] = nil
|
||||
}
|
||||
b.pool = nil
|
||||
return err
|
||||
}
|
|
@ -0,0 +1,464 @@
|
|||
package smb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"hash"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/cenkalti/backoff/v4"
|
||||
"github.com/hirochachacha/go-smb2"
|
||||
"github.com/restic/restic/internal/backend"
|
||||
"github.com/restic/restic/internal/backend/layout"
|
||||
"github.com/restic/restic/internal/backend/limiter"
|
||||
"github.com/restic/restic/internal/backend/location"
|
||||
"github.com/restic/restic/internal/backend/util"
|
||||
"github.com/restic/restic/internal/debug"
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
)
|
||||
|
||||
// Parts of this code have been adapted from Rclone (https://github.com/rclone)
|
||||
// Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/
|
||||
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
// of this software and associated documentation files (the "Software"), to deal
|
||||
// in the Software without restriction, including without limitation the rights
|
||||
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
// copies of the Software, and to permit persons to whom the Software is
|
||||
// furnished to do so, subject to the following conditions:
|
||||
|
||||
// The above copyright notice and this permission notice shall be included in
|
||||
// all copies or substantial portions of the Software.
|
||||
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
// THE SOFTWARE.
|
||||
|
||||
// Backend stores data on an SMB endpoint.
|
||||
type Backend struct {
|
||||
Config
|
||||
layout.Layout
|
||||
util.Modes
|
||||
|
||||
sessions int32
|
||||
poolMu sync.Mutex
|
||||
pool []*conn
|
||||
drain *time.Timer // used to drain the pool when we stop using the connections
|
||||
}
|
||||
|
||||
// make sure that *Backend implements backend.Backend
|
||||
var _ backend.Backend = &Backend{}
|
||||
|
||||
func NewFactory() location.Factory {
|
||||
return location.NewLimitedBackendFactory("smb", ParseConfig, location.NoPassword, limiter.WrapBackendConstructor(Create), limiter.WrapBackendConstructor(Open))
|
||||
}
|
||||
|
||||
const (
|
||||
defaultLayout = "default"
|
||||
)
|
||||
|
||||
func open(ctx context.Context, cfg Config) (*Backend, error) {
|
||||
|
||||
l, err := layout.ParseLayout(ctx, &layout.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
b := &Backend{
|
||||
Config: cfg,
|
||||
Layout: l,
|
||||
}
|
||||
|
||||
debug.Log("open, config %#v", cfg)
|
||||
|
||||
// set the pool drainer timer going
|
||||
if b.Config.IdleTimeout > 0 {
|
||||
b.drain = time.AfterFunc(b.Config.IdleTimeout, func() { _ = b.drainPool() })
|
||||
}
|
||||
|
||||
cn, err := b.getConnection(ctx, b.ShareName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer b.putConnection(cn)
|
||||
|
||||
stat, err := cn.smbShare.Stat(l.Filename(backend.Handle{Type: restic.ConfigFile}))
|
||||
m := util.DeriveModesFromFileInfo(stat, err)
|
||||
debug.Log("using (%03O file, %03O dir) permissions", m.File, m.Dir)
|
||||
|
||||
b.Modes = m
|
||||
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// Open opens the local backend as specified by config.
|
||||
func Open(ctx context.Context, cfg Config) (*Backend, error) {
|
||||
debug.Log("open local backend at %v (layout %q)", cfg.Path, cfg.Layout)
|
||||
return open(ctx, cfg)
|
||||
}
|
||||
|
||||
// Create creates all the necessary files and directories for a new local
|
||||
// backend at dir. Afterwards a new config blob should be created.
|
||||
func Create(ctx context.Context, cfg Config) (*Backend, error) {
|
||||
debug.Log("create local backend at %v (layout %q)", cfg.Path, cfg.Layout)
|
||||
|
||||
b, err := open(ctx, cfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cn, err := b.getConnection(ctx, cfg.ShareName)
|
||||
if err != nil {
|
||||
return b, err
|
||||
}
|
||||
defer b.putConnection(cn)
|
||||
|
||||
// test if config file already exists
|
||||
_, err = cn.smbShare.Lstat(b.Filename(backend.Handle{Type: restic.ConfigFile}))
|
||||
if err == nil {
|
||||
return nil, errors.New("config file already exists")
|
||||
}
|
||||
|
||||
// create paths for data and refs
|
||||
for _, d := range b.Paths() {
|
||||
err := cn.smbShare.MkdirAll(d, b.Modes.Dir)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
}
|
||||
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (b *Backend) Connections() uint {
|
||||
return b.Config.Connections
|
||||
}
|
||||
|
||||
// Location returns this backend's location (the directory name).
|
||||
func (b *Backend) Location() string {
|
||||
return b.Join(b.ShareName, b.Path)
|
||||
}
|
||||
|
||||
// Hasher may return a hash function for calculating a content hash for the backend
|
||||
func (b *Backend) Hasher() hash.Hash {
|
||||
return nil
|
||||
}
|
||||
|
||||
// HasAtomicReplace returns whether Save() can atomically replace files
|
||||
func (b *Backend) HasAtomicReplace() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// IsNotExist returns true if the error is caused by a non existing file.
|
||||
func (b *Backend) IsNotExist(err error) bool {
|
||||
return errors.Is(err, os.ErrNotExist)
|
||||
}
|
||||
|
||||
// Join combines path components with slashes.
|
||||
func (b *Backend) Join(p ...string) string {
|
||||
return path.Join(p...)
|
||||
}
|
||||
|
||||
// Save stores data in the backend at the handle.
|
||||
func (b *Backend) Save(ctx context.Context, h backend.Handle, rd backend.RewindReader) (err error) {
|
||||
filename := b.Filename(h)
|
||||
tmpFilename := filename + "-restic-temp-" + tempSuffix()
|
||||
dir := filepath.Dir(tmpFilename)
|
||||
|
||||
defer func() {
|
||||
// Mark non-retriable errors as such
|
||||
if errors.Is(err, syscall.ENOSPC) || os.IsPermission(err) {
|
||||
err = backoff.Permanent(err)
|
||||
}
|
||||
}()
|
||||
|
||||
b.addSession() // Show session in use
|
||||
defer b.removeSession()
|
||||
|
||||
cn, err := b.getConnection(ctx, b.ShareName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer b.putConnection(cn)
|
||||
|
||||
// create new file
|
||||
f, err := cn.smbShare.OpenFile(tmpFilename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
|
||||
|
||||
if b.IsNotExist(err) {
|
||||
debug.Log("error %v: creating dir", err)
|
||||
|
||||
// error is caused by a missing directory, try to create it
|
||||
mkdirErr := cn.smbShare.MkdirAll(dir, b.Modes.Dir)
|
||||
if mkdirErr != nil {
|
||||
debug.Log("error creating dir %v: %v", dir, mkdirErr)
|
||||
} else {
|
||||
// try again
|
||||
f, err = cn.smbShare.OpenFile(tmpFilename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
defer func(f *smb2.File) {
|
||||
if err != nil {
|
||||
_ = f.Close() // Double Close is harmless.
|
||||
// Remove after Rename is harmless: we embed the final name in the
|
||||
// temporary's name and no other goroutine will get the same data to
|
||||
// Save, so the temporary name should never be reused by another
|
||||
// goroutine.
|
||||
_ = cn.smbShare.Remove(f.Name())
|
||||
}
|
||||
}(f)
|
||||
|
||||
// save data, then sync
|
||||
wbytes, err := io.Copy(f, rd)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
// sanity check
|
||||
if wbytes != rd.Length() {
|
||||
return errors.Errorf("wrote %d bytes instead of the expected %d bytes", wbytes, rd.Length())
|
||||
}
|
||||
|
||||
// Ignore error if filesystem does not support fsync.
|
||||
// In this case the sync call is on the smb client's file.
|
||||
err = f.Sync()
|
||||
syncNotSup := err != nil && (errors.Is(err, syscall.ENOTSUP))
|
||||
if err != nil && !syncNotSup {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
// Close, then rename. Windows doesn't like the reverse order.
|
||||
if err = f.Close(); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
if err = cn.smbShare.Rename(f.Name(), filename); err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
// try to mark file as read-only to avoid accidential modifications
|
||||
// ignore if the operation fails as some filesystems don't allow the chmod call
|
||||
// e.g. exfat and network file systems with certain mount options
|
||||
err = cn.setFileReadonly(filename, b.Modes.File)
|
||||
if err != nil && !os.IsPermission(err) {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// set file to readonly
|
||||
func (cn *conn) setFileReadonly(f string, mode os.FileMode) error {
|
||||
return cn.smbShare.Chmod(f, mode&^0222)
|
||||
}
|
||||
|
||||
// Load runs fn with a reader that yields the contents of the file at h at the
|
||||
// given offset.
|
||||
func (b *Backend) Load(ctx context.Context, h backend.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
|
||||
return util.DefaultLoad(ctx, h, length, offset, b.openReader, fn)
|
||||
}
|
||||
|
||||
func (b *Backend) openReader(ctx context.Context, h backend.Handle, length int, offset int64) (io.ReadCloser, error) {
|
||||
b.addSession() // Show session in use
|
||||
defer b.removeSession()
|
||||
cn, err := b.getConnection(ctx, b.ShareName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer b.putConnection(cn)
|
||||
|
||||
f, err := cn.smbShare.Open(b.Filename(h))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if offset > 0 {
|
||||
_, err = f.Seek(offset, 0)
|
||||
if err != nil {
|
||||
_ = f.Close()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if length > 0 {
|
||||
return backend.LimitReadCloser(f, int64(length)), nil
|
||||
}
|
||||
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// Stat returns information about a blob.
|
||||
func (b *Backend) Stat(ctx context.Context, h backend.Handle) (backend.FileInfo, error) {
|
||||
cn, err := b.getConnection(ctx, b.ShareName)
|
||||
if err != nil {
|
||||
return backend.FileInfo{}, err
|
||||
}
|
||||
defer b.putConnection(cn)
|
||||
|
||||
fi, err := cn.smbShare.Stat(b.Filename(h))
|
||||
if err != nil {
|
||||
return backend.FileInfo{}, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return backend.FileInfo{Size: fi.Size(), Name: h.Name}, nil
|
||||
}
|
||||
|
||||
// Remove removes the blob with the given name and type.
|
||||
func (b *Backend) Remove(ctx context.Context, h backend.Handle) error {
|
||||
fn := b.Filename(h)
|
||||
|
||||
cn, err := b.getConnection(ctx, b.ShareName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer b.putConnection(cn)
|
||||
|
||||
// reset read-only flag
|
||||
err = cn.smbShare.Chmod(fn, 0666)
|
||||
if err != nil && !os.IsPermission(err) {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return cn.smbShare.Remove(fn)
|
||||
}
|
||||
|
||||
// List runs fn for each file in the backend which has the type t. When an
|
||||
// error occurs (or fn returns an error), List stops and returns it.
|
||||
func (b *Backend) List(ctx context.Context, t restic.FileType, fn func(backend.FileInfo) error) (err error) {
|
||||
cn, err := b.getConnection(ctx, b.ShareName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer b.putConnection(cn)
|
||||
|
||||
basedir, subdirs := b.Basedir(t)
|
||||
if subdirs {
|
||||
err = b.visitDirs(ctx, cn, basedir, fn)
|
||||
} else {
|
||||
err = b.visitFiles(ctx, cn, basedir, fn, false)
|
||||
}
|
||||
|
||||
if b.IsNotExist(err) {
|
||||
debug.Log("ignoring non-existing directory")
|
||||
return nil
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// The following two functions are like filepath.Walk, but visit only one or
|
||||
// two levels of directory structure (including dir itself as the first level).
|
||||
// Also, visitDirs assumes it sees a directory full of directories, while
|
||||
// visitFiles wants a directory full or regular files.
|
||||
func (b *Backend) visitDirs(ctx context.Context, cn *conn, dir string, fn func(backend.FileInfo) error) error {
|
||||
d, err := cn.smbShare.Open(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sub, err := d.Readdirnames(-1)
|
||||
if err != nil {
|
||||
// ignore subsequent errors
|
||||
_ = d.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
err = d.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, f := range sub {
|
||||
err = b.visitFiles(ctx, cn, filepath.Join(dir, f), fn, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func (b *Backend) visitFiles(ctx context.Context, cn *conn, dir string, fn func(backend.FileInfo) error, ignoreNotADirectory bool) error {
|
||||
d, err := cn.smbShare.Open(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if ignoreNotADirectory {
|
||||
fi, err := d.Stat()
|
||||
if err != nil || !fi.IsDir() {
|
||||
// ignore subsequent errors
|
||||
_ = d.Close()
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
sub, err := d.Readdir(-1)
|
||||
if err != nil {
|
||||
// ignore subsequent errors
|
||||
_ = d.Close()
|
||||
return err
|
||||
}
|
||||
|
||||
err = d.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, fi := range sub {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
default:
|
||||
}
|
||||
|
||||
err := fn(backend.FileInfo{
|
||||
Name: fi.Name(),
|
||||
Size: fi.Size(),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete removes the repository and all files.
|
||||
func (b *Backend) Delete(ctx context.Context) error {
|
||||
cn, err := b.getConnection(ctx, b.ShareName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer b.putConnection(cn)
|
||||
return cn.smbShare.RemoveAll(b.Location())
|
||||
}
|
||||
|
||||
// Close closes all open files.
|
||||
func (b *Backend) Close() error {
|
||||
err := b.drainPool()
|
||||
return err
|
||||
}
|
||||
|
||||
// tempSuffix generates a random string suffix that should be sufficiently long
|
||||
// to avoid accidental conflicts.
|
||||
func tempSuffix() string {
|
||||
var nonce [16]byte
|
||||
_, err := rand.Read(nonce[:])
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return hex.EncodeToString(nonce[:])
|
||||
}
|
|
@ -0,0 +1,58 @@
|
|||
package smb_test
|
||||
|
||||
import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/restic/restic/internal/backend/smb"
|
||||
"github.com/restic/restic/internal/backend/test"
|
||||
"github.com/restic/restic/internal/options"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
)
|
||||
|
||||
func newTestSuite(t testing.TB) *test.Suite[smb.Config] {
|
||||
return &test.Suite[smb.Config]{
|
||||
// NewConfig returns a config for a new temporary backend that will be used in tests.
|
||||
NewConfig: func() (*smb.Config, error) {
|
||||
|
||||
cfg := smb.NewConfig()
|
||||
cfg.Host = "127.0.0.1"
|
||||
cfg.User = "smbuser"
|
||||
cfg.ShareName = cfg.User
|
||||
cfg.Path = "Repo-" + uuid.New().String()
|
||||
cfg.Password = options.NewSecretString("mGoWwqvgdnwtmh07")
|
||||
cfg.Connections = smb.DefaultConnections
|
||||
timeout := smb.DefaultIdleTimeout
|
||||
cfg.IdleTimeout = timeout
|
||||
domain := os.Getenv("RESTIC_SMB_DOMAIN")
|
||||
if domain == "" {
|
||||
cfg.Domain = smb.DefaultDomain
|
||||
}
|
||||
|
||||
t.Logf("create new backend at %v", cfg.Host+"/"+cfg.ShareName)
|
||||
|
||||
return &cfg, nil
|
||||
},
|
||||
|
||||
Factory: smb.NewFactory(),
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendSMB(t *testing.T) {
|
||||
if !rtest.RunSMBTest {
|
||||
t.Skip("Skipping smb tests")
|
||||
}
|
||||
t.Logf("run tests")
|
||||
|
||||
newTestSuite(t).RunTests(t)
|
||||
}
|
||||
|
||||
func BenchmarkBackendSMB(t *testing.B) {
|
||||
if !rtest.RunSMBTest {
|
||||
t.Skip("Skipping smb tests")
|
||||
}
|
||||
t.Logf("run benchmarks")
|
||||
|
||||
newTestSuite(t).RunBenchmarks(t)
|
||||
}
|
|
@ -165,7 +165,8 @@ func (c *Cache) Clear(t restic.FileType, valid restic.IDSet) error {
|
|||
continue
|
||||
}
|
||||
|
||||
if err = fs.Remove(c.filename(backend.Handle{Type: t, Name: id.String()})); err != nil {
|
||||
// ignore ErrNotExist to gracefully handle multiple processes running Clear() concurrently
|
||||
if err = fs.Remove(c.filename(backend.Handle{Type: t, Name: id.String()})); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
|
@ -106,9 +106,9 @@ func (c *Checker) LoadSnapshots(ctx context.Context) error {
|
|||
return err
|
||||
}
|
||||
|
||||
func computePackTypes(ctx context.Context, idx restic.MasterIndex) map[restic.ID]restic.BlobType {
|
||||
func computePackTypes(ctx context.Context, idx restic.MasterIndex) (map[restic.ID]restic.BlobType, error) {
|
||||
packs := make(map[restic.ID]restic.BlobType)
|
||||
idx.Each(ctx, func(pb restic.PackedBlob) {
|
||||
err := idx.Each(ctx, func(pb restic.PackedBlob) {
|
||||
tpe, exists := packs[pb.PackID]
|
||||
if exists {
|
||||
if pb.Type != tpe {
|
||||
|
@ -119,7 +119,7 @@ func computePackTypes(ctx context.Context, idx restic.MasterIndex) map[restic.ID
|
|||
}
|
||||
packs[pb.PackID] = tpe
|
||||
})
|
||||
return packs
|
||||
return packs, err
|
||||
}
|
||||
|
||||
// LoadIndex loads all index files.
|
||||
|
@ -169,7 +169,7 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e
|
|||
|
||||
debug.Log("process blobs")
|
||||
cnt := 0
|
||||
index.Each(ctx, func(blob restic.PackedBlob) {
|
||||
err = index.Each(ctx, func(blob restic.PackedBlob) {
|
||||
cnt++
|
||||
|
||||
if _, ok := packToIndex[blob.PackID]; !ok {
|
||||
|
@ -179,7 +179,7 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e
|
|||
})
|
||||
|
||||
debug.Log("%d blobs processed", cnt)
|
||||
return nil
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
errs = append(errs, err)
|
||||
|
@ -193,8 +193,14 @@ func (c *Checker) LoadIndex(ctx context.Context, p *progress.Counter) (hints []e
|
|||
}
|
||||
|
||||
// compute pack size using index entries
|
||||
c.packs = pack.Size(ctx, c.masterIndex, false)
|
||||
packTypes := computePackTypes(ctx, c.masterIndex)
|
||||
c.packs, err = pack.Size(ctx, c.masterIndex, false)
|
||||
if err != nil {
|
||||
return hints, append(errs, err)
|
||||
}
|
||||
packTypes, err := computePackTypes(ctx, c.masterIndex)
|
||||
if err != nil {
|
||||
return hints, append(errs, err)
|
||||
}
|
||||
|
||||
debug.Log("checking for duplicate packs")
|
||||
for packID := range c.packs {
|
||||
|
@ -484,7 +490,7 @@ func (c *Checker) checkTree(id restic.ID, tree *restic.Tree) (errs []error) {
|
|||
}
|
||||
|
||||
// UnusedBlobs returns all blobs that have never been referenced.
|
||||
func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles) {
|
||||
func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles, err error) {
|
||||
if !c.trackUnused {
|
||||
panic("only works when tracking blob references")
|
||||
}
|
||||
|
@ -495,7 +501,7 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles) {
|
|||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
c.repo.Index().Each(ctx, func(blob restic.PackedBlob) {
|
||||
err = c.repo.Index().Each(ctx, func(blob restic.PackedBlob) {
|
||||
h := restic.BlobHandle{ID: blob.ID, Type: blob.Type}
|
||||
if !c.blobRefs.M.Has(h) {
|
||||
debug.Log("blob %v not referenced", h)
|
||||
|
@ -503,7 +509,7 @@ func (c *Checker) UnusedBlobs(ctx context.Context) (blobs restic.BlobHandles) {
|
|||
}
|
||||
})
|
||||
|
||||
return blobs
|
||||
return blobs, err
|
||||
}
|
||||
|
||||
// CountPacks returns the number of packs in the repository.
|
||||
|
|
|
@ -180,7 +180,8 @@ func TestUnreferencedBlobs(t *testing.T) {
|
|||
test.OKs(t, checkPacks(chkr))
|
||||
test.OKs(t, checkStruct(chkr))
|
||||
|
||||
blobs := chkr.UnusedBlobs(context.TODO())
|
||||
blobs, err := chkr.UnusedBlobs(context.TODO())
|
||||
test.OK(t, err)
|
||||
sort.Sort(blobs)
|
||||
|
||||
test.Equals(t, unusedBlobsBySnapshot, blobs)
|
||||
|
|
|
@ -8,7 +8,7 @@ import (
|
|||
)
|
||||
|
||||
// TestCheckRepo runs the checker on repo.
|
||||
func TestCheckRepo(t testing.TB, repo restic.Repository) {
|
||||
func TestCheckRepo(t testing.TB, repo restic.Repository, skipStructure bool) {
|
||||
chkr := New(repo, true)
|
||||
|
||||
hints, errs := chkr.LoadIndex(context.TODO(), nil)
|
||||
|
@ -33,18 +33,23 @@ func TestCheckRepo(t testing.TB, repo restic.Repository) {
|
|||
t.Error(err)
|
||||
}
|
||||
|
||||
// structure
|
||||
errChan = make(chan error)
|
||||
go chkr.Structure(context.TODO(), nil, errChan)
|
||||
if !skipStructure {
|
||||
// structure
|
||||
errChan = make(chan error)
|
||||
go chkr.Structure(context.TODO(), nil, errChan)
|
||||
|
||||
for err := range errChan {
|
||||
t.Error(err)
|
||||
}
|
||||
for err := range errChan {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
// unused blobs
|
||||
blobs := chkr.UnusedBlobs(context.TODO())
|
||||
if len(blobs) > 0 {
|
||||
t.Errorf("unused blobs found: %v", blobs)
|
||||
// unused blobs
|
||||
blobs, err := chkr.UnusedBlobs(context.TODO())
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if len(blobs) > 0 {
|
||||
t.Errorf("unused blobs found: %v", blobs)
|
||||
}
|
||||
}
|
||||
|
||||
// read data
|
||||
|
|
|
@ -190,7 +190,7 @@ func (e *vssError) Error() string {
|
|||
return fmt.Sprintf("VSS error: %s: %s (%#x)", e.text, e.hresult.Str(), e.hresult)
|
||||
}
|
||||
|
||||
// VssError encapsulates errors returned from calling VSS api.
|
||||
// vssTextError encapsulates errors returned from calling VSS api.
|
||||
type vssTextError struct {
|
||||
text string
|
||||
}
|
||||
|
|
|
@ -5,6 +5,7 @@ import (
|
|||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
|
@ -69,11 +70,9 @@ func (idx *Index) addToPacks(id restic.ID) int {
|
|||
return len(idx.packs) - 1
|
||||
}
|
||||
|
||||
const maxuint32 = 1<<32 - 1
|
||||
|
||||
func (idx *Index) store(packIndex int, blob restic.Blob) {
|
||||
// assert that offset and length fit into uint32!
|
||||
if blob.Offset > maxuint32 || blob.Length > maxuint32 || blob.UncompressedLength > maxuint32 {
|
||||
if blob.Offset > math.MaxUint32 || blob.Length > math.MaxUint32 || blob.UncompressedLength > math.MaxUint32 {
|
||||
panic("offset or length does not fit in uint32. You have packs > 4GB!")
|
||||
}
|
||||
|
||||
|
@ -219,7 +218,7 @@ func (idx *Index) AddToSupersedes(ids ...restic.ID) error {
|
|||
|
||||
// Each passes all blobs known to the index to the callback fn. This blocks any
|
||||
// modification of the index.
|
||||
func (idx *Index) Each(ctx context.Context, fn func(restic.PackedBlob)) {
|
||||
func (idx *Index) Each(ctx context.Context, fn func(restic.PackedBlob)) error {
|
||||
idx.m.Lock()
|
||||
defer idx.m.Unlock()
|
||||
|
||||
|
@ -233,6 +232,7 @@ func (idx *Index) Each(ctx context.Context, fn func(restic.PackedBlob)) {
|
|||
return true
|
||||
})
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
type EachByPackResult struct {
|
||||
|
|
|
@ -339,7 +339,7 @@ func TestIndexUnserialize(t *testing.T) {
|
|||
|
||||
rtest.Equals(t, oldIdx, idx.Supersedes())
|
||||
|
||||
blobs := listPack(idx, exampleLookupTest.packID)
|
||||
blobs := listPack(t, idx, exampleLookupTest.packID)
|
||||
if len(blobs) != len(exampleLookupTest.blobs) {
|
||||
t.Fatalf("expected %d blobs in pack, got %d", len(exampleLookupTest.blobs), len(blobs))
|
||||
}
|
||||
|
@ -356,12 +356,12 @@ func TestIndexUnserialize(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func listPack(idx *index.Index, id restic.ID) (pbs []restic.PackedBlob) {
|
||||
idx.Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||
func listPack(t testing.TB, idx *index.Index, id restic.ID) (pbs []restic.PackedBlob) {
|
||||
rtest.OK(t, idx.Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||
if pb.PackID.Equal(id) {
|
||||
pbs = append(pbs, pb)
|
||||
}
|
||||
})
|
||||
}))
|
||||
return pbs
|
||||
}
|
||||
|
||||
|
|
|
@ -223,13 +223,16 @@ func (mi *MasterIndex) finalizeFullIndexes() []*Index {
|
|||
|
||||
// Each runs fn on all blobs known to the index. When the context is cancelled,
|
||||
// the index iteration return immediately. This blocks any modification of the index.
|
||||
func (mi *MasterIndex) Each(ctx context.Context, fn func(restic.PackedBlob)) {
|
||||
func (mi *MasterIndex) Each(ctx context.Context, fn func(restic.PackedBlob)) error {
|
||||
mi.idxMutex.RLock()
|
||||
defer mi.idxMutex.RUnlock()
|
||||
|
||||
for _, idx := range mi.idx {
|
||||
idx.Each(ctx, fn)
|
||||
if err := idx.Each(ctx, fn); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MergeFinalIndexes merges all final indexes together.
|
||||
|
@ -320,6 +323,9 @@ func (mi *MasterIndex) Save(ctx context.Context, repo restic.Repository, exclude
|
|||
newIndex = NewIndex()
|
||||
}
|
||||
}
|
||||
if wgCtx.Err() != nil {
|
||||
return wgCtx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
err := newIndex.AddToSupersedes(extraObsolete...)
|
||||
|
@ -426,10 +432,6 @@ func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan
|
|||
defer close(out)
|
||||
// only resort a part of the index to keep the memory overhead bounded
|
||||
for i := byte(0); i < 16; i++ {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
packBlob := make(map[restic.ID][]restic.Blob)
|
||||
for pack := range packs {
|
||||
if pack[0]&0xf == i {
|
||||
|
@ -439,11 +441,14 @@ func (mi *MasterIndex) ListPacks(ctx context.Context, packs restic.IDSet) <-chan
|
|||
if len(packBlob) == 0 {
|
||||
continue
|
||||
}
|
||||
mi.Each(ctx, func(pb restic.PackedBlob) {
|
||||
err := mi.Each(ctx, func(pb restic.PackedBlob) {
|
||||
if packs.Has(pb.PackID) && pb.PackID[0]&0xf == i {
|
||||
packBlob[pb.PackID] = append(packBlob[pb.PackID], pb.Blob)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// pass on packs
|
||||
for packID, pbs := range packBlob {
|
||||
|
|
|
@ -166,9 +166,9 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
|
|||
rtest.Equals(t, 1, idxCount)
|
||||
|
||||
blobCount := 0
|
||||
mIdx.Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||
rtest.OK(t, mIdx.Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||
blobCount++
|
||||
})
|
||||
}))
|
||||
rtest.Equals(t, 2, blobCount)
|
||||
|
||||
blobs := mIdx.Lookup(bhInIdx1)
|
||||
|
@ -198,9 +198,9 @@ func TestMasterMergeFinalIndexes(t *testing.T) {
|
|||
rtest.Equals(t, []restic.PackedBlob{blob2}, blobs)
|
||||
|
||||
blobCount = 0
|
||||
mIdx.Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||
rtest.OK(t, mIdx.Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||
blobCount++
|
||||
})
|
||||
}))
|
||||
rtest.Equals(t, 2, blobCount)
|
||||
}
|
||||
|
||||
|
@ -319,9 +319,9 @@ func BenchmarkMasterIndexEach(b *testing.B) {
|
|||
|
||||
for i := 0; i < b.N; i++ {
|
||||
entries := 0
|
||||
mIdx.Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||
rtest.OK(b, mIdx.Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||
entries++
|
||||
})
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -211,6 +211,11 @@ func (o Options) Apply(ns string, dst interface{}) error {
|
|||
|
||||
v.Field(i).SetInt(int64(d))
|
||||
|
||||
case "SecretString":
|
||||
ss := NewSecretString(value)
|
||||
|
||||
v.Field(i).Set(reflect.ValueOf(ss))
|
||||
|
||||
default:
|
||||
panic("type " + v.Type().Field(i).Type.Name() + " not handled")
|
||||
}
|
||||
|
|
|
@ -389,10 +389,10 @@ func CalculateHeaderSize(blobs []restic.Blob) int {
|
|||
// If onlyHdr is set to true, only the size of the header is returned
|
||||
// Note that this function only gives correct sizes, if there are no
|
||||
// duplicates in the index.
|
||||
func Size(ctx context.Context, mi restic.MasterIndex, onlyHdr bool) map[restic.ID]int64 {
|
||||
func Size(ctx context.Context, mi restic.MasterIndex, onlyHdr bool) (map[restic.ID]int64, error) {
|
||||
packSize := make(map[restic.ID]int64)
|
||||
|
||||
mi.Each(ctx, func(blob restic.PackedBlob) {
|
||||
err := mi.Each(ctx, func(blob restic.PackedBlob) {
|
||||
size, ok := packSize[blob.PackID]
|
||||
if !ok {
|
||||
size = headerSize
|
||||
|
@ -403,5 +403,5 @@ func Size(ctx context.Context, mi restic.MasterIndex, onlyHdr bool) map[restic.I
|
|||
packSize[blob.PackID] = size + int64(CalculateEntrySize(blob.Blob))
|
||||
})
|
||||
|
||||
return packSize
|
||||
return packSize, err
|
||||
}
|
||||
|
|
|
@ -0,0 +1,638 @@
|
|||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
|
||||
"github.com/restic/restic/internal/errors"
|
||||
"github.com/restic/restic/internal/index"
|
||||
"github.com/restic/restic/internal/pack"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
)
|
||||
|
||||
var ErrIndexIncomplete = errors.Fatal("index is not complete")
|
||||
var ErrPacksMissing = errors.Fatal("packs from index missing in repo")
|
||||
var ErrSizeNotMatching = errors.Fatal("pack size does not match calculated size from index")
|
||||
|
||||
// PruneOptions collects all options for the cleanup command.
|
||||
type PruneOptions struct {
|
||||
DryRun bool
|
||||
UnsafeRecovery bool
|
||||
|
||||
MaxUnusedBytes func(used uint64) (unused uint64) // calculates the number of unused bytes after repacking, according to MaxUnused
|
||||
MaxRepackBytes uint64
|
||||
|
||||
RepackCachableOnly bool
|
||||
RepackSmall bool
|
||||
RepackUncompressed bool
|
||||
}
|
||||
|
||||
type PruneStats struct {
|
||||
Blobs struct {
|
||||
Used uint
|
||||
Duplicate uint
|
||||
Unused uint
|
||||
Remove uint
|
||||
Repack uint
|
||||
Repackrm uint
|
||||
}
|
||||
Size struct {
|
||||
Used uint64
|
||||
Duplicate uint64
|
||||
Unused uint64
|
||||
Remove uint64
|
||||
Repack uint64
|
||||
Repackrm uint64
|
||||
Unref uint64
|
||||
Uncompressed uint64
|
||||
}
|
||||
Packs struct {
|
||||
Used uint
|
||||
Unused uint
|
||||
PartlyUsed uint
|
||||
Unref uint
|
||||
Keep uint
|
||||
Repack uint
|
||||
Remove uint
|
||||
}
|
||||
}
|
||||
|
||||
type PrunePlan struct {
|
||||
removePacksFirst restic.IDSet // packs to remove first (unreferenced packs)
|
||||
repackPacks restic.IDSet // packs to repack
|
||||
keepBlobs restic.CountedBlobSet // blobs to keep during repacking
|
||||
removePacks restic.IDSet // packs to remove
|
||||
ignorePacks restic.IDSet // packs to ignore when rebuilding the index
|
||||
|
||||
repo restic.Repository
|
||||
stats PruneStats
|
||||
opts PruneOptions
|
||||
}
|
||||
|
||||
type packInfo struct {
|
||||
usedBlobs uint
|
||||
unusedBlobs uint
|
||||
usedSize uint64
|
||||
unusedSize uint64
|
||||
tpe restic.BlobType
|
||||
uncompressed bool
|
||||
}
|
||||
|
||||
type packInfoWithID struct {
|
||||
ID restic.ID
|
||||
packInfo
|
||||
mustCompress bool
|
||||
}
|
||||
|
||||
// PlanPrune selects which files to rewrite and which to delete and which blobs to keep.
|
||||
// Also some summary statistics are returned.
|
||||
func PlanPrune(ctx context.Context, opts PruneOptions, repo restic.Repository, getUsedBlobs func(ctx context.Context, repo restic.Repository) (usedBlobs restic.CountedBlobSet, err error), printer progress.Printer) (*PrunePlan, error) {
|
||||
var stats PruneStats
|
||||
|
||||
if opts.UnsafeRecovery {
|
||||
// prevent repacking data to make sure users cannot get stuck.
|
||||
opts.MaxRepackBytes = 0
|
||||
}
|
||||
if repo.Connections() < 2 {
|
||||
return nil, fmt.Errorf("prune requires a backend connection limit of at least two")
|
||||
}
|
||||
if repo.Config().Version < 2 && opts.RepackUncompressed {
|
||||
return nil, fmt.Errorf("compression requires at least repository format version 2")
|
||||
}
|
||||
|
||||
usedBlobs, err := getUsedBlobs(ctx, repo)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
printer.P("searching used packs...\n")
|
||||
keepBlobs, indexPack, err := packInfoFromIndex(ctx, repo.Index(), usedBlobs, &stats, printer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
printer.P("collecting packs for deletion and repacking\n")
|
||||
plan, err := decidePackAction(ctx, opts, repo, indexPack, &stats, printer)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(plan.repackPacks) != 0 {
|
||||
blobCount := keepBlobs.Len()
|
||||
// when repacking, we do not want to keep blobs which are
|
||||
// already contained in kept packs, so delete them from keepBlobs
|
||||
err := repo.Index().Each(ctx, func(blob restic.PackedBlob) {
|
||||
if plan.removePacks.Has(blob.PackID) || plan.repackPacks.Has(blob.PackID) {
|
||||
return
|
||||
}
|
||||
keepBlobs.Delete(blob.BlobHandle)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if keepBlobs.Len() < blobCount/2 {
|
||||
// replace with copy to shrink map to necessary size if there's a chance to benefit
|
||||
keepBlobs = keepBlobs.Copy()
|
||||
}
|
||||
} else {
|
||||
// keepBlobs is only needed if packs are repacked
|
||||
keepBlobs = nil
|
||||
}
|
||||
plan.keepBlobs = keepBlobs
|
||||
|
||||
plan.repo = repo
|
||||
plan.stats = stats
|
||||
plan.opts = opts
|
||||
|
||||
return &plan, nil
|
||||
}
|
||||
|
||||
func packInfoFromIndex(ctx context.Context, idx restic.MasterIndex, usedBlobs restic.CountedBlobSet, stats *PruneStats, printer progress.Printer) (restic.CountedBlobSet, map[restic.ID]packInfo, error) {
|
||||
// iterate over all blobs in index to find out which blobs are duplicates
|
||||
// The counter in usedBlobs describes how many instances of the blob exist in the repository index
|
||||
// Thus 0 == blob is missing, 1 == blob exists once, >= 2 == duplicates exist
|
||||
err := idx.Each(ctx, func(blob restic.PackedBlob) {
|
||||
bh := blob.BlobHandle
|
||||
count, ok := usedBlobs[bh]
|
||||
if ok {
|
||||
if count < math.MaxUint8 {
|
||||
// don't overflow, but saturate count at 255
|
||||
// this can lead to a non-optimal pack selection, but won't cause
|
||||
// problems otherwise
|
||||
count++
|
||||
}
|
||||
|
||||
usedBlobs[bh] = count
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Check if all used blobs have been found in index
|
||||
missingBlobs := restic.NewBlobSet()
|
||||
for bh, count := range usedBlobs {
|
||||
if count == 0 {
|
||||
// blob does not exist in any pack files
|
||||
missingBlobs.Insert(bh)
|
||||
}
|
||||
}
|
||||
|
||||
if len(missingBlobs) != 0 {
|
||||
printer.E("%v not found in the index\n\n"+
|
||||
"Integrity check failed: Data seems to be missing.\n"+
|
||||
"Will not start prune to prevent (additional) data loss!\n"+
|
||||
"Please report this error (along with the output of the 'prune' run) at\n"+
|
||||
"https://github.com/restic/restic/issues/new/choose\n", missingBlobs)
|
||||
return nil, nil, ErrIndexIncomplete
|
||||
}
|
||||
|
||||
indexPack := make(map[restic.ID]packInfo)
|
||||
|
||||
// save computed pack header size
|
||||
sz, err := pack.Size(ctx, idx, true)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
for pid, hdrSize := range sz {
|
||||
// initialize tpe with NumBlobTypes to indicate it's not set
|
||||
indexPack[pid] = packInfo{tpe: restic.NumBlobTypes, usedSize: uint64(hdrSize)}
|
||||
}
|
||||
|
||||
hasDuplicates := false
|
||||
// iterate over all blobs in index to generate packInfo
|
||||
err = idx.Each(ctx, func(blob restic.PackedBlob) {
|
||||
ip := indexPack[blob.PackID]
|
||||
|
||||
// Set blob type if not yet set
|
||||
if ip.tpe == restic.NumBlobTypes {
|
||||
ip.tpe = blob.Type
|
||||
}
|
||||
|
||||
// mark mixed packs with "Invalid blob type"
|
||||
if ip.tpe != blob.Type {
|
||||
ip.tpe = restic.InvalidBlob
|
||||
}
|
||||
|
||||
bh := blob.BlobHandle
|
||||
size := uint64(blob.Length)
|
||||
dupCount := usedBlobs[bh]
|
||||
switch {
|
||||
case dupCount >= 2:
|
||||
hasDuplicates = true
|
||||
// mark as unused for now, we will later on select one copy
|
||||
ip.unusedSize += size
|
||||
ip.unusedBlobs++
|
||||
|
||||
// count as duplicate, will later on change one copy to be counted as used
|
||||
stats.Size.Duplicate += size
|
||||
stats.Blobs.Duplicate++
|
||||
case dupCount == 1: // used blob, not duplicate
|
||||
ip.usedSize += size
|
||||
ip.usedBlobs++
|
||||
|
||||
stats.Size.Used += size
|
||||
stats.Blobs.Used++
|
||||
default: // unused blob
|
||||
ip.unusedSize += size
|
||||
ip.unusedBlobs++
|
||||
|
||||
stats.Size.Unused += size
|
||||
stats.Blobs.Unused++
|
||||
}
|
||||
if !blob.IsCompressed() {
|
||||
ip.uncompressed = true
|
||||
}
|
||||
// update indexPack
|
||||
indexPack[blob.PackID] = ip
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// if duplicate blobs exist, those will be set to either "used" or "unused":
|
||||
// - mark only one occurrence of duplicate blobs as used
|
||||
// - if there are already some used blobs in a pack, possibly mark duplicates in this pack as "used"
|
||||
// - if there are no used blobs in a pack, possibly mark duplicates as "unused"
|
||||
if hasDuplicates {
|
||||
// iterate again over all blobs in index (this is pretty cheap, all in-mem)
|
||||
err = idx.Each(ctx, func(blob restic.PackedBlob) {
|
||||
bh := blob.BlobHandle
|
||||
count, ok := usedBlobs[bh]
|
||||
// skip non-duplicate, aka. normal blobs
|
||||
// count == 0 is used to mark that this was a duplicate blob with only a single occurrence remaining
|
||||
if !ok || count == 1 {
|
||||
return
|
||||
}
|
||||
|
||||
ip := indexPack[blob.PackID]
|
||||
size := uint64(blob.Length)
|
||||
switch {
|
||||
case ip.usedBlobs > 0, count == 0:
|
||||
// other used blobs in pack or "last" occurrence -> transition to used
|
||||
ip.usedSize += size
|
||||
ip.usedBlobs++
|
||||
ip.unusedSize -= size
|
||||
ip.unusedBlobs--
|
||||
// same for the global statistics
|
||||
stats.Size.Used += size
|
||||
stats.Blobs.Used++
|
||||
stats.Size.Duplicate -= size
|
||||
stats.Blobs.Duplicate--
|
||||
// let other occurrences remain marked as unused
|
||||
usedBlobs[bh] = 1
|
||||
default:
|
||||
// remain unused and decrease counter
|
||||
count--
|
||||
if count == 1 {
|
||||
// setting count to 1 would lead to forgetting that this blob had duplicates
|
||||
// thus use the special value zero. This will select the last instance of the blob for keeping.
|
||||
count = 0
|
||||
}
|
||||
usedBlobs[bh] = count
|
||||
}
|
||||
// update indexPack
|
||||
indexPack[blob.PackID] = ip
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Sanity check. If no duplicates exist, all blobs have value 1. After handling
|
||||
// duplicates, this also applies to duplicates.
|
||||
for _, count := range usedBlobs {
|
||||
if count != 1 {
|
||||
panic("internal error during blob selection")
|
||||
}
|
||||
}
|
||||
|
||||
return usedBlobs, indexPack, nil
|
||||
}
|
||||
|
||||
func decidePackAction(ctx context.Context, opts PruneOptions, repo restic.Repository, indexPack map[restic.ID]packInfo, stats *PruneStats, printer progress.Printer) (PrunePlan, error) {
|
||||
removePacksFirst := restic.NewIDSet()
|
||||
removePacks := restic.NewIDSet()
|
||||
repackPacks := restic.NewIDSet()
|
||||
|
||||
var repackCandidates []packInfoWithID
|
||||
var repackSmallCandidates []packInfoWithID
|
||||
repoVersion := repo.Config().Version
|
||||
// only repack very small files by default
|
||||
targetPackSize := repo.PackSize() / 25
|
||||
if opts.RepackSmall {
|
||||
// consider files with at least 80% of the target size as large enough
|
||||
targetPackSize = repo.PackSize() / 5 * 4
|
||||
}
|
||||
|
||||
// loop over all packs and decide what to do
|
||||
bar := printer.NewCounter("packs processed")
|
||||
bar.SetMax(uint64(len(indexPack)))
|
||||
err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
|
||||
p, ok := indexPack[id]
|
||||
if !ok {
|
||||
// Pack was not referenced in index and is not used => immediately remove!
|
||||
printer.V("will remove pack %v as it is unused and not indexed\n", id.Str())
|
||||
removePacksFirst.Insert(id)
|
||||
stats.Size.Unref += uint64(packSize)
|
||||
return nil
|
||||
}
|
||||
|
||||
if p.unusedSize+p.usedSize != uint64(packSize) && p.usedBlobs != 0 {
|
||||
// Pack size does not fit and pack is needed => error
|
||||
// If the pack is not needed, this is no error, the pack can
|
||||
// and will be simply removed, see below.
|
||||
printer.E("pack %s: calculated size %d does not match real size %d\nRun 'restic repair index'.\n",
|
||||
id.Str(), p.unusedSize+p.usedSize, packSize)
|
||||
return ErrSizeNotMatching
|
||||
}
|
||||
|
||||
// statistics
|
||||
switch {
|
||||
case p.usedBlobs == 0:
|
||||
stats.Packs.Unused++
|
||||
case p.unusedBlobs == 0:
|
||||
stats.Packs.Used++
|
||||
default:
|
||||
stats.Packs.PartlyUsed++
|
||||
}
|
||||
|
||||
if p.uncompressed {
|
||||
stats.Size.Uncompressed += p.unusedSize + p.usedSize
|
||||
}
|
||||
mustCompress := false
|
||||
if repoVersion >= 2 {
|
||||
// repo v2: always repack tree blobs if uncompressed
|
||||
// compress data blobs if requested
|
||||
mustCompress = (p.tpe == restic.TreeBlob || opts.RepackUncompressed) && p.uncompressed
|
||||
}
|
||||
|
||||
// decide what to do
|
||||
switch {
|
||||
case p.usedBlobs == 0:
|
||||
// All blobs in pack are no longer used => remove pack!
|
||||
removePacks.Insert(id)
|
||||
stats.Blobs.Remove += p.unusedBlobs
|
||||
stats.Size.Remove += p.unusedSize
|
||||
|
||||
case opts.RepackCachableOnly && p.tpe == restic.DataBlob:
|
||||
// if this is a data pack and --repack-cacheable-only is set => keep pack!
|
||||
stats.Packs.Keep++
|
||||
|
||||
case p.unusedBlobs == 0 && p.tpe != restic.InvalidBlob && !mustCompress:
|
||||
if packSize >= int64(targetPackSize) {
|
||||
// All blobs in pack are used and not mixed => keep pack!
|
||||
stats.Packs.Keep++
|
||||
} else {
|
||||
repackSmallCandidates = append(repackSmallCandidates, packInfoWithID{ID: id, packInfo: p, mustCompress: mustCompress})
|
||||
}
|
||||
|
||||
default:
|
||||
// all other packs are candidates for repacking
|
||||
repackCandidates = append(repackCandidates, packInfoWithID{ID: id, packInfo: p, mustCompress: mustCompress})
|
||||
}
|
||||
|
||||
delete(indexPack, id)
|
||||
bar.Add(1)
|
||||
return nil
|
||||
})
|
||||
bar.Done()
|
||||
if err != nil {
|
||||
return PrunePlan{}, err
|
||||
}
|
||||
|
||||
// At this point indexPacks contains only missing packs!
|
||||
|
||||
// missing packs that are not needed can be ignored
|
||||
ignorePacks := restic.NewIDSet()
|
||||
for id, p := range indexPack {
|
||||
if p.usedBlobs == 0 {
|
||||
ignorePacks.Insert(id)
|
||||
stats.Blobs.Remove += p.unusedBlobs
|
||||
stats.Size.Remove += p.unusedSize
|
||||
delete(indexPack, id)
|
||||
}
|
||||
}
|
||||
|
||||
if len(indexPack) != 0 {
|
||||
printer.E("The index references %d needed pack files which are missing from the repository:\n", len(indexPack))
|
||||
for id := range indexPack {
|
||||
printer.E(" %v\n", id)
|
||||
}
|
||||
return PrunePlan{}, ErrPacksMissing
|
||||
}
|
||||
if len(ignorePacks) != 0 {
|
||||
printer.E("Missing but unneeded pack files are referenced in the index, will be repaired\n")
|
||||
for id := range ignorePacks {
|
||||
printer.E("will forget missing pack file %v\n", id)
|
||||
}
|
||||
}
|
||||
|
||||
if len(repackSmallCandidates) < 10 {
|
||||
// too few small files to be worth the trouble, this also prevents endlessly repacking
|
||||
// if there is just a single pack file below the target size
|
||||
stats.Packs.Keep += uint(len(repackSmallCandidates))
|
||||
} else {
|
||||
repackCandidates = append(repackCandidates, repackSmallCandidates...)
|
||||
}
|
||||
|
||||
// Sort repackCandidates such that packs with highest ratio unused/used space are picked first.
|
||||
// This is equivalent to sorting by unused / total space.
|
||||
// Instead of unused[i] / used[i] > unused[j] / used[j] we use
|
||||
// unused[i] * used[j] > unused[j] * used[i] as uint32*uint32 < uint64
|
||||
// Moreover packs containing trees and too small packs are sorted to the beginning
|
||||
sort.Slice(repackCandidates, func(i, j int) bool {
|
||||
pi := repackCandidates[i].packInfo
|
||||
pj := repackCandidates[j].packInfo
|
||||
switch {
|
||||
case pi.tpe != restic.DataBlob && pj.tpe == restic.DataBlob:
|
||||
return true
|
||||
case pj.tpe != restic.DataBlob && pi.tpe == restic.DataBlob:
|
||||
return false
|
||||
case pi.unusedSize+pi.usedSize < uint64(targetPackSize) && pj.unusedSize+pj.usedSize >= uint64(targetPackSize):
|
||||
return true
|
||||
case pj.unusedSize+pj.usedSize < uint64(targetPackSize) && pi.unusedSize+pi.usedSize >= uint64(targetPackSize):
|
||||
return false
|
||||
}
|
||||
return pi.unusedSize*pj.usedSize > pj.unusedSize*pi.usedSize
|
||||
})
|
||||
|
||||
repack := func(id restic.ID, p packInfo) {
|
||||
repackPacks.Insert(id)
|
||||
stats.Blobs.Repack += p.unusedBlobs + p.usedBlobs
|
||||
stats.Size.Repack += p.unusedSize + p.usedSize
|
||||
stats.Blobs.Repackrm += p.unusedBlobs
|
||||
stats.Size.Repackrm += p.unusedSize
|
||||
if p.uncompressed {
|
||||
stats.Size.Uncompressed -= p.unusedSize + p.usedSize
|
||||
}
|
||||
}
|
||||
|
||||
// calculate limit for number of unused bytes in the repo after repacking
|
||||
maxUnusedSizeAfter := opts.MaxUnusedBytes(stats.Size.Used)
|
||||
|
||||
for _, p := range repackCandidates {
|
||||
reachedUnusedSizeAfter := (stats.Size.Unused-stats.Size.Remove-stats.Size.Repackrm < maxUnusedSizeAfter)
|
||||
reachedRepackSize := stats.Size.Repack+p.unusedSize+p.usedSize >= opts.MaxRepackBytes
|
||||
packIsLargeEnough := p.unusedSize+p.usedSize >= uint64(targetPackSize)
|
||||
|
||||
switch {
|
||||
case reachedRepackSize:
|
||||
stats.Packs.Keep++
|
||||
|
||||
case p.tpe != restic.DataBlob, p.mustCompress:
|
||||
// repacking non-data packs / uncompressed-trees is only limited by repackSize
|
||||
repack(p.ID, p.packInfo)
|
||||
|
||||
case reachedUnusedSizeAfter && packIsLargeEnough:
|
||||
// for all other packs stop repacking if tolerated unused size is reached.
|
||||
stats.Packs.Keep++
|
||||
|
||||
default:
|
||||
repack(p.ID, p.packInfo)
|
||||
}
|
||||
}
|
||||
|
||||
stats.Packs.Unref = uint(len(removePacksFirst))
|
||||
stats.Packs.Repack = uint(len(repackPacks))
|
||||
stats.Packs.Remove = uint(len(removePacks))
|
||||
|
||||
if repo.Config().Version < 2 {
|
||||
// compression not supported for repository format version 1
|
||||
stats.Size.Uncompressed = 0
|
||||
}
|
||||
|
||||
return PrunePlan{removePacksFirst: removePacksFirst,
|
||||
removePacks: removePacks,
|
||||
repackPacks: repackPacks,
|
||||
ignorePacks: ignorePacks,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (plan *PrunePlan) Stats() PruneStats {
|
||||
return plan.stats
|
||||
}
|
||||
|
||||
// Execute does the actual pruning:
|
||||
// - remove unreferenced packs first
|
||||
// - repack given pack files while keeping the given blobs
|
||||
// - rebuild the index while ignoring all files that will be deleted
|
||||
// - delete the files
|
||||
// plan.removePacks and plan.ignorePacks are modified in this function.
|
||||
func (plan *PrunePlan) Execute(ctx context.Context, printer progress.Printer) (err error) {
|
||||
if plan.opts.DryRun {
|
||||
printer.V("Repeated prune dry-runs can report slightly different amounts of data to keep or repack. This is expected behavior.\n\n")
|
||||
if len(plan.removePacksFirst) > 0 {
|
||||
printer.V("Would have removed the following unreferenced packs:\n%v\n\n", plan.removePacksFirst)
|
||||
}
|
||||
printer.V("Would have repacked and removed the following packs:\n%v\n\n", plan.repackPacks)
|
||||
printer.V("Would have removed the following no longer used packs:\n%v\n\n", plan.removePacks)
|
||||
// Always quit here if DryRun was set!
|
||||
return nil
|
||||
}
|
||||
|
||||
repo := plan.repo
|
||||
// make sure the plan can only be used once
|
||||
plan.repo = nil
|
||||
|
||||
// unreferenced packs can be safely deleted first
|
||||
if len(plan.removePacksFirst) != 0 {
|
||||
printer.P("deleting unreferenced packs\n")
|
||||
_ = deleteFiles(ctx, true, repo, plan.removePacksFirst, restic.PackFile, printer)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if len(plan.repackPacks) != 0 {
|
||||
printer.P("repacking packs\n")
|
||||
bar := printer.NewCounter("packs repacked")
|
||||
bar.SetMax(uint64(len(plan.repackPacks)))
|
||||
_, err := Repack(ctx, repo, repo, plan.repackPacks, plan.keepBlobs, bar)
|
||||
bar.Done()
|
||||
if err != nil {
|
||||
return errors.Fatal(err.Error())
|
||||
}
|
||||
|
||||
// Also remove repacked packs
|
||||
plan.removePacks.Merge(plan.repackPacks)
|
||||
|
||||
if len(plan.keepBlobs) != 0 {
|
||||
printer.E("%v was not repacked\n\n"+
|
||||
"Integrity check failed.\n"+
|
||||
"Please report this error (along with the output of the 'prune' run) at\n"+
|
||||
"https://github.com/restic/restic/issues/new/choose\n", plan.keepBlobs)
|
||||
return errors.Fatal("internal error: blobs were not repacked")
|
||||
}
|
||||
|
||||
// allow GC of the blob set
|
||||
plan.keepBlobs = nil
|
||||
}
|
||||
|
||||
if len(plan.ignorePacks) == 0 {
|
||||
plan.ignorePacks = plan.removePacks
|
||||
} else {
|
||||
plan.ignorePacks.Merge(plan.removePacks)
|
||||
}
|
||||
|
||||
if plan.opts.UnsafeRecovery {
|
||||
printer.P("deleting index files\n")
|
||||
indexFiles := repo.Index().(*index.MasterIndex).IDs()
|
||||
err = deleteFiles(ctx, false, repo, indexFiles, restic.IndexFile, printer)
|
||||
if err != nil {
|
||||
return errors.Fatalf("%s", err)
|
||||
}
|
||||
} else if len(plan.ignorePacks) != 0 {
|
||||
err = rebuildIndexFiles(ctx, repo, plan.ignorePacks, nil, false, printer)
|
||||
if err != nil {
|
||||
return errors.Fatalf("%s", err)
|
||||
}
|
||||
}
|
||||
|
||||
if len(plan.removePacks) != 0 {
|
||||
printer.P("removing %d old packs\n", len(plan.removePacks))
|
||||
_ = deleteFiles(ctx, true, repo, plan.removePacks, restic.PackFile, printer)
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if plan.opts.UnsafeRecovery {
|
||||
err = rebuildIndexFiles(ctx, repo, plan.ignorePacks, nil, true, printer)
|
||||
if err != nil {
|
||||
return errors.Fatalf("%s", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// drop outdated in-memory index
|
||||
repo.ClearIndex()
|
||||
|
||||
printer.P("done\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
// deleteFiles deletes the given fileList of fileType in parallel
|
||||
// if ignoreError=true, it will print a warning if there was an error, else it will abort.
|
||||
func deleteFiles(ctx context.Context, ignoreError bool, repo restic.Repository, fileList restic.IDSet, fileType restic.FileType, printer progress.Printer) error {
|
||||
bar := printer.NewCounter("files deleted")
|
||||
defer bar.Done()
|
||||
|
||||
return restic.ParallelRemove(ctx, repo, fileList, fileType, func(id restic.ID, err error) error {
|
||||
if err != nil {
|
||||
printer.E("unable to remove %v/%v from the repository\n", fileType, id)
|
||||
if !ignoreError {
|
||||
return err
|
||||
}
|
||||
}
|
||||
printer.VV("removed %v/%v\n", fileType, id)
|
||||
return nil
|
||||
}, bar)
|
||||
}
|
|
@ -0,0 +1,105 @@
|
|||
package repository_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/checker"
|
||||
"github.com/restic/restic/internal/repository"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
func testPrune(t *testing.T, opts repository.PruneOptions, errOnUnused bool) {
|
||||
repo := repository.TestRepository(t).(*repository.Repository)
|
||||
createRandomBlobs(t, repo, 4, 0.5, true)
|
||||
createRandomBlobs(t, repo, 5, 0.5, true)
|
||||
keep, _ := selectBlobs(t, repo, 0.5)
|
||||
|
||||
var wg errgroup.Group
|
||||
repo.StartPackUploader(context.TODO(), &wg)
|
||||
// duplicate a few blobs to exercise those code paths
|
||||
for blob := range keep {
|
||||
buf, err := repo.LoadBlob(context.TODO(), blob.Type, blob.ID, nil)
|
||||
rtest.OK(t, err)
|
||||
_, _, _, err = repo.SaveBlob(context.TODO(), blob.Type, buf, blob.ID, true)
|
||||
rtest.OK(t, err)
|
||||
}
|
||||
rtest.OK(t, repo.Flush(context.TODO()))
|
||||
|
||||
plan, err := repository.PlanPrune(context.TODO(), opts, repo, func(ctx context.Context, repo restic.Repository) (usedBlobs restic.CountedBlobSet, err error) {
|
||||
return restic.NewCountedBlobSet(keep.List()...), nil
|
||||
}, &progress.NoopPrinter{})
|
||||
rtest.OK(t, err)
|
||||
|
||||
rtest.OK(t, plan.Execute(context.TODO(), &progress.NoopPrinter{}))
|
||||
|
||||
repo = repository.TestOpenBackend(t, repo.Backend()).(*repository.Repository)
|
||||
checker.TestCheckRepo(t, repo, true)
|
||||
|
||||
if errOnUnused {
|
||||
existing := listBlobs(repo)
|
||||
rtest.Assert(t, existing.Equals(keep), "unexpected blobs, wanted %v got %v", keep, existing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrune(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
opts repository.PruneOptions
|
||||
errOnUnused bool
|
||||
}{
|
||||
{
|
||||
name: "0",
|
||||
opts: repository.PruneOptions{
|
||||
MaxRepackBytes: math.MaxUint64,
|
||||
MaxUnusedBytes: func(used uint64) (unused uint64) { return 0 },
|
||||
},
|
||||
errOnUnused: true,
|
||||
},
|
||||
{
|
||||
name: "50",
|
||||
opts: repository.PruneOptions{
|
||||
MaxRepackBytes: math.MaxUint64,
|
||||
MaxUnusedBytes: func(used uint64) (unused uint64) { return used / 2 },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "unlimited",
|
||||
opts: repository.PruneOptions{
|
||||
MaxRepackBytes: math.MaxUint64,
|
||||
MaxUnusedBytes: func(used uint64) (unused uint64) { return math.MaxUint64 },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "cachableonly",
|
||||
opts: repository.PruneOptions{
|
||||
MaxRepackBytes: math.MaxUint64,
|
||||
MaxUnusedBytes: func(used uint64) (unused uint64) { return used / 20 },
|
||||
RepackCachableOnly: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "small",
|
||||
opts: repository.PruneOptions{
|
||||
MaxRepackBytes: math.MaxUint64,
|
||||
MaxUnusedBytes: func(used uint64) (unused uint64) { return math.MaxUint64 },
|
||||
RepackSmall: true,
|
||||
},
|
||||
errOnUnused: true,
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
testPrune(t, test.opts, test.errOnUnused)
|
||||
})
|
||||
t.Run(test.name+"-recovery", func(t *testing.T) {
|
||||
opts := test.opts
|
||||
opts.UnsafeRecovery = true
|
||||
// unsafeNoSpaceRecovery does not repack partially used pack files
|
||||
testPrune(t, opts, false)
|
||||
})
|
||||
}
|
||||
}
|
|
@ -72,7 +72,7 @@ func repack(ctx context.Context, repo restic.Repository, dstRepo restic.Reposito
|
|||
return wgCtx.Err()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
return wgCtx.Err()
|
||||
})
|
||||
|
||||
worker := func() error {
|
||||
|
|
|
@ -18,7 +18,7 @@ func randomSize(min, max int) int {
|
|||
return rand.Intn(max-min) + min
|
||||
}
|
||||
|
||||
func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData float32) {
|
||||
func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData float32, smallBlobs bool) {
|
||||
var wg errgroup.Group
|
||||
repo.StartPackUploader(context.TODO(), &wg)
|
||||
|
||||
|
@ -30,7 +30,11 @@ func createRandomBlobs(t testing.TB, repo restic.Repository, blobs int, pData fl
|
|||
|
||||
if rand.Float32() < pData {
|
||||
tpe = restic.DataBlob
|
||||
length = randomSize(10*1024, 1024*1024) // 10KiB to 1MiB of data
|
||||
if smallBlobs {
|
||||
length = randomSize(1*1024, 20*1024) // 1KiB to 20KiB of data
|
||||
} else {
|
||||
length = randomSize(10*1024, 1024*1024) // 10KiB to 1MiB of data
|
||||
}
|
||||
} else {
|
||||
tpe = restic.TreeBlob
|
||||
length = randomSize(1*1024, 20*1024) // 1KiB to 20KiB
|
||||
|
@ -121,8 +125,12 @@ func selectBlobs(t *testing.T, repo restic.Repository, p float32) (list1, list2
|
|||
}
|
||||
|
||||
func listPacks(t *testing.T, repo restic.Lister) restic.IDSet {
|
||||
return listFiles(t, repo, restic.PackFile)
|
||||
}
|
||||
|
||||
func listFiles(t *testing.T, repo restic.Lister, tpe backend.FileType) restic.IDSet {
|
||||
list := restic.NewIDSet()
|
||||
err := repo.List(context.TODO(), restic.PackFile, func(id restic.ID, size int64) error {
|
||||
err := repo.List(context.TODO(), tpe, func(id restic.ID, size int64) error {
|
||||
list.Insert(id)
|
||||
return nil
|
||||
})
|
||||
|
@ -166,12 +174,6 @@ func repack(t *testing.T, repo restic.Repository, packs restic.IDSet, blobs rest
|
|||
}
|
||||
}
|
||||
|
||||
func flush(t *testing.T, repo restic.Repository) {
|
||||
if err := repo.Flush(context.TODO()); err != nil {
|
||||
t.Fatalf("repo.SaveIndex() %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func rebuildIndex(t *testing.T, repo restic.Repository) {
|
||||
err := repo.SetIndex(index.NewMasterIndex())
|
||||
rtest.OK(t, err)
|
||||
|
@ -219,7 +221,9 @@ func testRepack(t *testing.T, version uint) {
|
|||
rand.Seed(seed)
|
||||
t.Logf("rand seed is %v", seed)
|
||||
|
||||
createRandomBlobs(t, repo, 100, 0.7)
|
||||
// add a small amount of blobs twice to create multiple pack files
|
||||
createRandomBlobs(t, repo, 10, 0.7, false)
|
||||
createRandomBlobs(t, repo, 10, 0.7, false)
|
||||
|
||||
packsBefore := listPacks(t, repo)
|
||||
|
||||
|
@ -233,8 +237,6 @@ func testRepack(t *testing.T, version uint) {
|
|||
packsBefore, packsAfter)
|
||||
}
|
||||
|
||||
flush(t, repo)
|
||||
|
||||
removeBlobs, keepBlobs := selectBlobs(t, repo, 0.2)
|
||||
|
||||
removePacks := findPacksForBlobs(t, repo, removeBlobs)
|
||||
|
@ -302,8 +304,9 @@ func testRepackCopy(t *testing.T, version uint) {
|
|||
rand.Seed(seed)
|
||||
t.Logf("rand seed is %v", seed)
|
||||
|
||||
createRandomBlobs(t, repo, 100, 0.7)
|
||||
flush(t, repo)
|
||||
// add a small amount of blobs twice to create multiple pack files
|
||||
createRandomBlobs(t, repo, 10, 0.7, false)
|
||||
createRandomBlobs(t, repo, 10, 0.7, false)
|
||||
|
||||
_, keepBlobs := selectBlobs(t, repo, 0.2)
|
||||
copyPacks := findPacksForBlobs(t, repo, keepBlobs)
|
||||
|
@ -343,7 +346,7 @@ func testRepackWrongBlob(t *testing.T, version uint) {
|
|||
rand.Seed(seed)
|
||||
t.Logf("rand seed is %v", seed)
|
||||
|
||||
createRandomBlobs(t, repo, 5, 0.7)
|
||||
createRandomBlobs(t, repo, 5, 0.7, false)
|
||||
createRandomWrongBlob(t, repo)
|
||||
|
||||
// just keep all blobs, but also rewrite every pack
|
||||
|
|
|
@ -0,0 +1,132 @@
|
|||
package repository
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/restic/restic/internal/index"
|
||||
"github.com/restic/restic/internal/pack"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
)
|
||||
|
||||
type RepairIndexOptions struct {
|
||||
ReadAllPacks bool
|
||||
}
|
||||
|
||||
func RepairIndex(ctx context.Context, repo *Repository, opts RepairIndexOptions, printer progress.Printer) error {
|
||||
var obsoleteIndexes restic.IDs
|
||||
packSizeFromList := make(map[restic.ID]int64)
|
||||
packSizeFromIndex := make(map[restic.ID]int64)
|
||||
removePacks := restic.NewIDSet()
|
||||
|
||||
if opts.ReadAllPacks {
|
||||
// get list of old index files but start with empty index
|
||||
err := repo.List(ctx, restic.IndexFile, func(id restic.ID, _ int64) error {
|
||||
obsoleteIndexes = append(obsoleteIndexes, id)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
printer.P("loading indexes...\n")
|
||||
mi := index.NewMasterIndex()
|
||||
err := index.ForAllIndexes(ctx, repo, repo, func(id restic.ID, idx *index.Index, _ bool, err error) error {
|
||||
if err != nil {
|
||||
printer.E("removing invalid index %v: %v\n", id, err)
|
||||
obsoleteIndexes = append(obsoleteIndexes, id)
|
||||
return nil
|
||||
}
|
||||
|
||||
mi.Insert(idx)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = mi.MergeFinalIndexes()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = repo.SetIndex(mi)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
packSizeFromIndex, err = pack.Size(ctx, repo.Index(), false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
printer.P("getting pack files to read...\n")
|
||||
err := repo.List(ctx, restic.PackFile, func(id restic.ID, packSize int64) error {
|
||||
size, ok := packSizeFromIndex[id]
|
||||
if !ok || size != packSize {
|
||||
// Pack was not referenced in index or size does not match
|
||||
packSizeFromList[id] = packSize
|
||||
removePacks.Insert(id)
|
||||
}
|
||||
if !ok {
|
||||
printer.E("adding pack file to index %v\n", id)
|
||||
} else if size != packSize {
|
||||
printer.E("reindexing pack file %v with unexpected size %v instead of %v\n", id, packSize, size)
|
||||
}
|
||||
delete(packSizeFromIndex, id)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for id := range packSizeFromIndex {
|
||||
// forget pack files that are referenced in the index but do not exist
|
||||
// when rebuilding the index
|
||||
removePacks.Insert(id)
|
||||
printer.E("removing not found pack file %v\n", id)
|
||||
}
|
||||
|
||||
if len(packSizeFromList) > 0 {
|
||||
printer.P("reading pack files\n")
|
||||
bar := printer.NewCounter("packs")
|
||||
bar.SetMax(uint64(len(packSizeFromList)))
|
||||
invalidFiles, err := repo.CreateIndexFromPacks(ctx, packSizeFromList, bar)
|
||||
bar.Done()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, id := range invalidFiles {
|
||||
printer.V("skipped incomplete pack file: %v\n", id)
|
||||
}
|
||||
}
|
||||
|
||||
err = rebuildIndexFiles(ctx, repo, removePacks, obsoleteIndexes, false, printer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// drop outdated in-memory index
|
||||
repo.ClearIndex()
|
||||
return nil
|
||||
}
|
||||
|
||||
func rebuildIndexFiles(ctx context.Context, repo restic.Repository, removePacks restic.IDSet, extraObsolete restic.IDs, skipDeletion bool, printer progress.Printer) error {
|
||||
printer.P("rebuilding index\n")
|
||||
|
||||
bar := printer.NewCounter("packs processed")
|
||||
return repo.Index().Save(ctx, repo, removePacks, extraObsolete, restic.MasterIndexSaveOpts{
|
||||
SaveProgress: bar,
|
||||
DeleteProgress: func() *progress.Counter {
|
||||
return printer.NewCounter("old indexes deleted")
|
||||
},
|
||||
DeleteReport: func(id restic.ID, err error) {
|
||||
if err != nil {
|
||||
printer.VV("failed to remove index %v: %v\n", id.String(), err)
|
||||
} else {
|
||||
printer.VV("removed index %v\n", id.String())
|
||||
}
|
||||
},
|
||||
SkipDeletion: skipDeletion,
|
||||
})
|
||||
}
|
|
@ -0,0 +1,79 @@
|
|||
package repository_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/restic/restic/internal/backend"
|
||||
"github.com/restic/restic/internal/checker"
|
||||
"github.com/restic/restic/internal/repository"
|
||||
"github.com/restic/restic/internal/restic"
|
||||
rtest "github.com/restic/restic/internal/test"
|
||||
"github.com/restic/restic/internal/ui/progress"
|
||||
)
|
||||
|
||||
func listIndex(t *testing.T, repo restic.Lister) restic.IDSet {
|
||||
return listFiles(t, repo, restic.IndexFile)
|
||||
}
|
||||
|
||||
func testRebuildIndex(t *testing.T, readAllPacks bool, damage func(t *testing.T, repo *repository.Repository)) {
|
||||
repo := repository.TestRepository(t).(*repository.Repository)
|
||||
createRandomBlobs(t, repo, 4, 0.5, true)
|
||||
createRandomBlobs(t, repo, 5, 0.5, true)
|
||||
indexes := listIndex(t, repo)
|
||||
t.Logf("old indexes %v", indexes)
|
||||
|
||||
damage(t, repo)
|
||||
|
||||
repo = repository.TestOpenBackend(t, repo.Backend()).(*repository.Repository)
|
||||
rtest.OK(t, repository.RepairIndex(context.TODO(), repo, repository.RepairIndexOptions{
|
||||
ReadAllPacks: readAllPacks,
|
||||
}, &progress.NoopPrinter{}))
|
||||
|
||||
newIndexes := listIndex(t, repo)
|
||||
old := indexes.Intersect(newIndexes)
|
||||
rtest.Assert(t, len(old) == 0, "expected old indexes to be removed, found %v", old)
|
||||
|
||||
checker.TestCheckRepo(t, repo, true)
|
||||
}
|
||||
|
||||
func TestRebuildIndex(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
damage func(t *testing.T, repo *repository.Repository)
|
||||
}{
|
||||
{
|
||||
"valid index",
|
||||
func(t *testing.T, repo *repository.Repository) {},
|
||||
},
|
||||
{
|
||||
"damaged index",
|
||||
func(t *testing.T, repo *repository.Repository) {
|
||||
index := listIndex(t, repo).List()[0]
|
||||
replaceFile(t, repo, backend.Handle{Type: restic.IndexFile, Name: index.String()}, func(b []byte) []byte {
|
||||
b[0] ^= 0xff
|
||||
return b
|
||||
})
|
||||
},
|
||||
},
|
||||
{
|
||||
"missing index",
|
||||
func(t *testing.T, repo *repository.Repository) {
|
||||
index := listIndex(t, repo).List()[0]
|
||||
rtest.OK(t, repo.Backend().Remove(context.TODO(), backend.Handle{Type: restic.IndexFile, Name: index.String()}))
|
||||
},
|
||||
},
|
||||
{
|
||||
"missing pack",
|
||||
func(t *testing.T, repo *repository.Repository) {
|
||||
pack := listPacks(t, repo).List()[0]
|
||||
rtest.OK(t, repo.Backend().Remove(context.TODO(), backend.Handle{Type: restic.PackFile, Name: pack.String()}))
|
||||
},
|
||||
},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
testRebuildIndex(t, false, test.damage)
|
||||
testRebuildIndex(t, true, test.damage)
|
||||
})
|
||||
}
|
||||
}
|
|
@ -60,19 +60,7 @@ func RepairPacks(ctx context.Context, repo restic.Repository, ids restic.IDSet,
|
|||
}
|
||||
|
||||
// remove salvaged packs from index
|
||||
printer.P("rebuilding index")
|
||||
|
||||
bar = printer.NewCounter("packs processed")
|
||||
err = repo.Index().Save(ctx, repo, ids, nil, restic.MasterIndexSaveOpts{
|
||||
SaveProgress: bar,
|
||||
DeleteProgress: func() *progress.Counter {
|
||||
return printer.NewCounter("old indexes deleted")
|
||||
},
|
||||
DeleteReport: func(id restic.ID, _ error) {
|
||||
printer.VV("removed index %v", id.String())
|
||||
},
|
||||
})
|
||||
|
||||
err = rebuildIndexFiles(ctx, repo, ids, nil, false, printer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -17,7 +17,7 @@ import (
|
|||
|
||||
func listBlobs(repo restic.Repository) restic.BlobSet {
|
||||
blobs := restic.NewBlobSet()
|
||||
repo.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||
_ = repo.Index().Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||
blobs.Insert(pb.BlobHandle)
|
||||
})
|
||||
return blobs
|
||||
|
@ -109,7 +109,7 @@ func testRepairBrokenPack(t *testing.T, version uint) {
|
|||
rand.Seed(seed)
|
||||
t.Logf("rand seed is %v", seed)
|
||||
|
||||
createRandomBlobs(t, repo, 5, 0.7)
|
||||
createRandomBlobs(t, repo, 5, 0.7, true)
|
||||
packsBefore := listPacks(t, repo)
|
||||
blobsBefore := listBlobs(repo)
|
||||
|
||||
|
|
|
@ -6,6 +6,7 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"runtime"
|
||||
"sort"
|
||||
|
@ -142,9 +143,6 @@ func (r *Repository) DisableAutoIndexUpdate() {
|
|||
// setConfig assigns the given config and updates the repository parameters accordingly
|
||||
func (r *Repository) setConfig(cfg restic.Config) {
|
||||
r.cfg = cfg
|
||||
if r.cfg.Version >= 2 {
|
||||
r.idx.MarkCompressed()
|
||||
}
|
||||
}
|
||||
|
||||
// Config returns the repository configuration.
|
||||
|
@ -637,9 +635,21 @@ func (r *Repository) Index() restic.MasterIndex {
|
|||
// SetIndex instructs the repository to use the given index.
|
||||
func (r *Repository) SetIndex(i restic.MasterIndex) error {
|
||||
r.idx = i.(*index.MasterIndex)
|
||||
r.configureIndex()
|
||||
return r.prepareCache()
|
||||
}
|
||||
|
||||
func (r *Repository) ClearIndex() {
|
||||
r.idx = index.NewMasterIndex()
|
||||
r.configureIndex()
|
||||
}
|
||||
|
||||
func (r *Repository) configureIndex() {
|
||||
if r.cfg.Version >= 2 {
|
||||
r.idx.MarkCompressed()
|
||||
}
|
||||
}
|
||||
|
||||
// LoadIndex loads all index files from the backend in parallel and stores them
|
||||
func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error {
|
||||
debug.Log("Loading index")
|
||||
|
@ -662,6 +672,9 @@ func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error {
|
|||
defer p.Done()
|
||||
}
|
||||
|
||||
// reset in-memory index before loading it from the repository
|
||||
r.ClearIndex()
|
||||
|
||||
err = index.ForAllIndexes(ctx, indexList, r, func(_ restic.ID, idx *index.Index, _ bool, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -691,15 +704,21 @@ func (r *Repository) LoadIndex(ctx context.Context, p *progress.Counter) error {
|
|||
defer cancel()
|
||||
|
||||
invalidIndex := false
|
||||
r.idx.Each(ctx, func(blob restic.PackedBlob) {
|
||||
err := r.idx.Each(ctx, func(blob restic.PackedBlob) {
|
||||
if blob.IsCompressed() {
|
||||
invalidIndex = true
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if invalidIndex {
|
||||
return errors.New("index uses feature not supported by repository version 1")
|
||||
}
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
// remove index files from the cache which have been removed in the repo
|
||||
return r.prepareCache()
|
||||
|
@ -917,6 +936,10 @@ func (r *Repository) Close() error {
|
|||
// occupies in the repo (compressed or not, including encryption overhead).
|
||||
func (r *Repository) SaveBlob(ctx context.Context, t restic.BlobType, buf []byte, id restic.ID, storeDuplicate bool) (newID restic.ID, known bool, size int, err error) {
|
||||
|
||||
if int64(len(buf)) > math.MaxUint32 {
|
||||
return restic.ID{}, false, 0, fmt.Errorf("blob is larger than 4GB")
|
||||
}
|
||||
|
||||
// compute plaintext hash if not already set
|
||||
if id.IsNull() {
|
||||
// Special case the hash calculation for all zero chunks. This is especially
|
||||
|
|
|
@ -242,8 +242,7 @@ func loadIndex(ctx context.Context, repo restic.LoaderUnpacked, id restic.ID) (*
|
|||
}
|
||||
|
||||
func TestRepositoryLoadUnpackedBroken(t *testing.T) {
|
||||
repo, cleanup := repository.TestFromFixture(t, repoFixture)
|
||||
defer cleanup()
|
||||
repo := repository.TestRepository(t)
|
||||
|
||||
data := rtest.Random(23, 12345)
|
||||
id := restic.Hash(data)
|
||||
|
@ -252,7 +251,7 @@ func TestRepositoryLoadUnpackedBroken(t *testing.T) {
|
|||
data[0] ^= 0xff
|
||||
|
||||
// store broken file
|
||||
err := repo.Backend().Save(context.TODO(), h, backend.NewByteReader(data, nil))
|
||||
err := repo.Backend().Save(context.TODO(), h, backend.NewByteReader(data, repo.Backend().Hasher()))
|
||||
rtest.OK(t, err)
|
||||
|
||||
// without a retry backend this will just return an error that the file is broken
|
||||
|
@ -371,13 +370,13 @@ func testRepositoryIncrementalIndex(t *testing.T, version uint) {
|
|||
idx, err := loadIndex(context.TODO(), repo, id)
|
||||
rtest.OK(t, err)
|
||||
|
||||
idx.Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||
rtest.OK(t, idx.Each(context.TODO(), func(pb restic.PackedBlob) {
|
||||
if _, ok := packEntries[pb.PackID]; !ok {
|
||||
packEntries[pb.PackID] = make(map[restic.ID]struct{})
|
||||
}
|
||||
|
||||
packEntries[pb.PackID][id] = struct{}{}
|
||||
})
|
||||
}))
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
|
|
|
@ -60,8 +60,11 @@ func TestRepositoryWithBackend(t testing.TB, be backend.Backend, version uint, o
|
|||
t.Fatalf("TestRepository(): new repo failed: %v", err)
|
||||
}
|
||||
|
||||
cfg := restic.TestCreateConfig(t, testChunkerPol, version)
|
||||
err = repo.init(context.TODO(), test.TestPassword, cfg)
|
||||
if version == 0 {
|
||||
version = restic.StableRepoVersion
|
||||
}
|
||||
pol := testChunkerPol
|
||||
err = repo.Init(context.TODO(), version, test.TestPassword, &pol)
|
||||
if err != nil {
|
||||
t.Fatalf("TestRepository(): initialize repo failed: %v", err)
|
||||
}
|
||||
|
|
|
@ -51,22 +51,6 @@ func CreateConfig(version uint) (Config, error) {
|
|||
return cfg, nil
|
||||
}
|
||||
|
||||
// TestCreateConfig creates a config for use within tests.
|
||||
func TestCreateConfig(t testing.TB, pol chunker.Pol, version uint) (cfg Config) {
|
||||
cfg.ChunkerPolynomial = pol
|
||||
|
||||
cfg.ID = NewRandomID().String()
|
||||
if version == 0 {
|
||||
version = StableRepoVersion
|
||||
}
|
||||
if version < MinRepoVersion || version > MaxRepoVersion {
|
||||
t.Fatalf("version %d is out of range", version)
|
||||
}
|
||||
cfg.Version = version
|
||||
|
||||
return cfg
|
||||
}
|
||||
|
||||
var checkPolynomial = true
|
||||
var checkPolynomialOnce sync.Once
|
||||
|
||||
|
|
|
@ -26,6 +26,7 @@ type Repository interface {
|
|||
|
||||
Index() MasterIndex
|
||||
LoadIndex(context.Context, *progress.Counter) error
|
||||
ClearIndex()
|
||||
SetIndex(MasterIndex) error
|
||||
LookupBlobSize(ID, BlobType) (uint, bool)
|
||||
|
||||
|
@ -102,8 +103,8 @@ type MasterIndex interface {
|
|||
Lookup(BlobHandle) []PackedBlob
|
||||
|
||||
// Each runs fn on all blobs known to the index. When the context is cancelled,
|
||||
// the index iteration return immediately. This blocks any modification of the index.
|
||||
Each(ctx context.Context, fn func(PackedBlob))
|
||||
// the index iteration returns immediately with ctx.Err(). This blocks any modification of the index.
|
||||
Each(ctx context.Context, fn func(PackedBlob)) error
|
||||
ListPacks(ctx context.Context, packs IDSet) <-chan PackBlobs
|
||||
|
||||
Save(ctx context.Context, repo Repository, excludePacks IDSet, extraObsolete IDs, opts MasterIndexSaveOpts) error
|
||||
|
|
|
@ -45,7 +45,7 @@ func TestCreateSnapshot(t *testing.T) {
|
|||
t.Fatalf("snapshot has zero tree ID")
|
||||
}
|
||||
|
||||
checker.TestCheckRepo(t, repo)
|
||||
checker.TestCheckRepo(t, repo, false)
|
||||
}
|
||||
|
||||
func BenchmarkTestCreateSnapshot(t *testing.B) {
|
||||
|
|
|
@ -13,6 +13,7 @@ var (
|
|||
TestTempDir = getStringVar("RESTIC_TEST_TMPDIR", "")
|
||||
RunIntegrationTest = getBoolVar("RESTIC_TEST_INTEGRATION", true)
|
||||
RunFuseTest = getBoolVar("RESTIC_TEST_FUSE", true)
|
||||
RunSMBTest = getBoolVar("RESTIC_TEST_SMB", true)
|
||||
TestSFTPPath = getStringVar("RESTIC_TEST_SFTPPATH", "/usr/lib/ssh:/usr/lib/openssh:/usr/libexec")
|
||||
TestWalkerPath = getStringVar("RESTIC_TEST_PATH", ".")
|
||||
BenchArchiveDirectory = getStringVar("RESTIC_BENCH_DIR", ".")
|
||||
|
|
|
@ -1,5 +1,7 @@
|
|||
package progress
|
||||
|
||||
import "testing"
|
||||
|
||||
// A Printer can can return a new counter or print messages
|
||||
// at different log levels.
|
||||
// It must be safe to call its methods from concurrent goroutines.
|
||||
|
@ -28,3 +30,36 @@ func (*NoopPrinter) P(_ string, _ ...interface{}) {}
|
|||
func (*NoopPrinter) V(_ string, _ ...interface{}) {}
|
||||
|
||||
func (*NoopPrinter) VV(_ string, _ ...interface{}) {}
|
||||
|
||||
// TestPrinter prints messages during testing
|
||||
type TestPrinter struct {
|
||||
t testing.TB
|
||||
}
|
||||
|
||||
func NewTestPrinter(t testing.TB) *TestPrinter {
|
||||
return &TestPrinter{
|
||||
t: t,
|
||||
}
|
||||
}
|
||||
|
||||
var _ Printer = (*TestPrinter)(nil)
|
||||
|
||||
func (p *TestPrinter) NewCounter(_ string) *Counter {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *TestPrinter) E(msg string, args ...interface{}) {
|
||||
p.t.Logf("error: "+msg, args...)
|
||||
}
|
||||
|
||||
func (p *TestPrinter) P(msg string, args ...interface{}) {
|
||||
p.t.Logf("print: "+msg, args...)
|
||||
}
|
||||
|
||||
func (p *TestPrinter) V(msg string, args ...interface{}) {
|
||||
p.t.Logf("verbose: "+msg, args...)
|
||||
}
|
||||
|
||||
func (p *TestPrinter) VV(msg string, args ...interface{}) {
|
||||
p.t.Logf("verbose2: "+msg, args...)
|
||||
}
|
||||
|
|
|
@ -13,7 +13,7 @@ import (
|
|||
// TestTree is used to construct a list of trees for testing the walker.
|
||||
type TestTree map[string]interface{}
|
||||
|
||||
// TestNode is used to test the walker.
|
||||
// TestFile is used to test the walker.
|
||||
type TestFile struct {
|
||||
Size uint64
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue