diff --git a/changelog/unreleased/pull-1735 b/changelog/unreleased/pull-1735 new file mode 100644 index 000000000..2cfd115d8 --- /dev/null +++ b/changelog/unreleased/pull-1735 @@ -0,0 +1,9 @@ +Enhancement: Allow keeping a time range of snaphots + +We've added the `--keep-within` option to the `forget` command. It instructs +restic to keep all snapshots within the given duration since the newest +snapshot. For example, running `restic forget --keep-within 5m7d` will keep all +snapshots which have been made in the five months and seven days since the +latest snapshot. + +https://github.com/restic/restic/pull/1735 diff --git a/cmd/restic/cmd_forget.go b/cmd/restic/cmd_forget.go index 79cc9f449..4afef1380 100644 --- a/cmd/restic/cmd_forget.go +++ b/cmd/restic/cmd_forget.go @@ -33,6 +33,7 @@ type ForgetOptions struct { Weekly int Monthly int Yearly int + Within restic.Duration KeepTags restic.TagLists Host string @@ -58,6 +59,7 @@ func init() { f.IntVarP(&forgetOptions.Weekly, "keep-weekly", "w", 0, "keep the last `n` weekly snapshots") f.IntVarP(&forgetOptions.Monthly, "keep-monthly", "m", 0, "keep the last `n` monthly snapshots") f.IntVarP(&forgetOptions.Yearly, "keep-yearly", "y", 0, "keep the last `n` yearly snapshots") + f.VarP(&forgetOptions.Within, "keep-within", "", "keep snapshots that were created within `duration` before the newest (e.g. 1y5m7d)") f.Var(&forgetOptions.KeepTags, "keep-tag", "keep snapshots with this `taglist` (can be specified multiple times)") // Sadly the commonly used shortcut `H` is already used. @@ -170,6 +172,7 @@ func runForget(opts ForgetOptions, gopts GlobalOptions, args []string) error { Weekly: opts.Weekly, Monthly: opts.Monthly, Yearly: opts.Yearly, + Within: opts.Within, Tags: opts.KeepTags, } diff --git a/doc/060_forget.rst b/doc/060_forget.rst index ab5274758..1ca614a79 100644 --- a/doc/060_forget.rst +++ b/doc/060_forget.rst @@ -159,6 +159,10 @@ The ``forget`` command accepts the following parameters: snapshots, only keep the last one for that year. - ``--keep-tag`` keep all snapshots which have all tags specified by this option (can be specified multiple times). +- ``--keep-within duration`` keep all snapshots which have been made within + the duration of the latest snapshot. ``duration`` needs to be a number of + years, months, and days, e.g. ``2y5m7d`` will keep all snapshots made in the + two years, five months, and seven days before the latest snapshot. Additionally, you can restrict removing snapshots to those which have a particular hostname with the ``--hostname`` parameter, or tags with the diff --git a/internal/restic/duration.go b/internal/restic/duration.go new file mode 100644 index 000000000..09289849b --- /dev/null +++ b/internal/restic/duration.go @@ -0,0 +1,131 @@ +package restic + +import ( + "fmt" + "strconv" + "strings" + "unicode" + + "github.com/restic/restic/internal/errors" +) + +// Duration is similar to time.Duration, except it only supports larger ranges +// like days, months, and years. +type Duration struct { + Days, Months, Years int +} + +func (d Duration) String() string { + var s string + if d.Years != 0 { + s += fmt.Sprintf("%dy", d.Years) + } + + if d.Months != 0 { + s += fmt.Sprintf("%dm", d.Months) + } + + if d.Days != 0 { + s += fmt.Sprintf("%dd", d.Days) + } + + return s +} + +func nextNumber(input string) (num int, rest string, err error) { + if len(input) == 0 { + return 0, "", nil + } + + var ( + n string + negative bool + ) + + if input[0] == '-' { + negative = true + input = input[1:] + } + + for i, s := range input { + if !unicode.IsNumber(s) { + rest = input[i:] + break + } + + n += string(s) + } + + if len(n) == 0 { + return 0, input, errors.New("no number found") + } + + num, err = strconv.Atoi(n) + if err != nil { + panic(err) + } + + if negative { + num = -num + } + + return num, rest, nil +} + +// ParseDuration parses a duration from a string. The format is: +// 6y5m234d +func ParseDuration(s string) (Duration, error) { + var ( + d Duration + num int + err error + ) + + s = strings.TrimSpace(s) + + for s != "" { + num, s, err = nextNumber(s) + if err != nil { + return Duration{}, err + } + + if len(s) == 0 { + return Duration{}, errors.Errorf("no unit found after number %d", num) + } + + switch s[0] { + case 'y': + d.Years = num + case 'm': + d.Months = num + case 'd': + d.Days = num + } + + s = s[1:] + } + + return d, nil +} + +// Set calls ParseDuration and updates d. +func (d *Duration) Set(s string) error { + v, err := ParseDuration(s) + if err != nil { + return err + } + + *d = v + return nil +} + +// Type returns the type of Duration, usable within github.com/spf13/pflag and +// in help texts. +func (d Duration) Type() string { + return "duration" +} + +// Zero returns true if the duration is empty (all values are set to zero). +func (d Duration) Zero() bool { + return d.Years == 0 && d.Months == 0 && d.Days == 0 +} diff --git a/internal/restic/duration_test.go b/internal/restic/duration_test.go new file mode 100644 index 000000000..0d5306069 --- /dev/null +++ b/internal/restic/duration_test.go @@ -0,0 +1,82 @@ +package restic + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestNextNumber(t *testing.T) { + var tests = []struct { + input string + num int + rest string + err bool + }{ + { + input: "3d", num: 3, rest: "d", + }, + { + input: "7m5d", num: 7, rest: "m5d", + }, + { + input: "-23y7m5d", num: -23, rest: "y7m5d", + }, + { + input: " 5d", num: 0, rest: " 5d", err: true, + }, + { + input: "5d ", num: 5, rest: "d ", + }, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + num, rest, err := nextNumber(test.input) + + if err != nil && !test.err { + t.Fatal(err) + } + + if num != test.num { + t.Errorf("wrong num, want %d, got %d", test.num, num) + } + + if rest != test.rest { + t.Errorf("wrong rest, want %q, got %q", test.rest, rest) + } + }) + } +} + +func TestParseDuration(t *testing.T) { + var tests = []struct { + input string + d Duration + output string + }{ + {"3d", Duration{Days: 3}, "3d"}, + {"7m5d", Duration{Months: 7, Days: 5}, "7m5d"}, + {"5d7m", Duration{Months: 7, Days: 5}, "7m5d"}, + {"-7m5d", Duration{Months: -7, Days: 5}, "-7m5d"}, + {"2y7m-5d", Duration{Years: 2, Months: 7, Days: -5}, "2y7m-5d"}, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + d, err := ParseDuration(test.input) + if err != nil { + t.Fatal(err) + } + + if !cmp.Equal(d, test.d) { + t.Error(cmp.Diff(test.d, d)) + } + + s := d.String() + if s != test.output { + t.Errorf("unexpected return of String(), want %q, got %q", test.output, s) + } + }) + } +} diff --git a/internal/restic/snapshot_policy.go b/internal/restic/snapshot_policy.go index 8dd7e5ed6..df142c0fb 100644 --- a/internal/restic/snapshot_policy.go +++ b/internal/restic/snapshot_policy.go @@ -16,6 +16,7 @@ type ExpirePolicy struct { Weekly int // keep the last n weekly snapshots Monthly int // keep the last n monthly snapshots Yearly int // keep the last n yearly snapshots + Within Duration // keep snapshots made within this duration Tags []TagList // keep all snapshots that include at least one of the tag lists. } @@ -40,12 +41,23 @@ func (e ExpirePolicy) String() (s string) { keeps = append(keeps, fmt.Sprintf("%d yearly", e.Yearly)) } - s = "keep the last " - for _, k := range keeps { - s += k + ", " + if len(keeps) > 0 { + s = fmt.Sprintf("keep the last %s snapshots", strings.Join(keeps, ", ")) + } + + if len(e.Tags) > 0 { + if s != "" { + s += " and " + } + s += fmt.Sprintf("all snapshots with tags %s", e.Tags) + } + + if !e.Within.Zero() { + if s != "" { + s += " and " + } + s += fmt.Sprintf("all snapshots within %s of the newest", e.Within) } - s = strings.Trim(s, ", ") - s += " snapshots" return s } @@ -97,6 +109,22 @@ func always(d time.Time, nr int) int { return nr } +// findLatestTimestamp returns the time stamp for the newest snapshot. +func findLatestTimestamp(list Snapshots) time.Time { + if len(list) == 0 { + panic("list of snapshots is empty") + } + + var latest time.Time + for _, sn := range list { + if sn.Time.After(latest) { + latest = sn.Time + } + } + + return latest +} + // ApplyPolicy returns the snapshots from list that are to be kept and removed // according to the policy p. list is sorted in the process. func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots) { @@ -123,6 +151,8 @@ func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots) { {p.Yearly, y, -1}, } + latest := findLatestTimestamp(list) + for nr, cur := range list { var keepSnap bool @@ -133,6 +163,14 @@ func ApplyPolicy(list Snapshots, p ExpirePolicy) (keep, remove Snapshots) { } } + // If the timestamp of the snapshot is within the range, then keep it. + if !p.Within.Zero() { + t := latest.AddDate(-p.Within.Years, -p.Within.Months, -p.Within.Days) + if cur.Time.After(t) { + keepSnap = true + } + } + // Now update the other buckets and see if they have some counts left. for i, b := range buckets { if b.Count > 0 { diff --git a/internal/restic/snapshot_policy_test.go b/internal/restic/snapshot_policy_test.go index 69111fab1..7ac89e809 100644 --- a/internal/restic/snapshot_policy_test.go +++ b/internal/restic/snapshot_policy_test.go @@ -21,6 +21,15 @@ func parseTimeUTC(s string) time.Time { return t.UTC() } +func parseDuration(s string) restic.Duration { + d, err := restic.ParseDuration(s) + if err != nil { + panic(err) + } + + return d +} + func TestExpireSnapshotOps(t *testing.T) { data := []struct { expectEmpty bool @@ -43,189 +52,194 @@ func TestExpireSnapshotOps(t *testing.T) { } } -var testExpireSnapshots = restic.Snapshots{ - {Time: parseTimeUTC("2014-09-01 10:20:30")}, - {Time: parseTimeUTC("2014-09-02 10:20:30")}, - {Time: parseTimeUTC("2014-09-05 10:20:30")}, - {Time: parseTimeUTC("2014-09-06 10:20:30")}, - {Time: parseTimeUTC("2014-09-08 10:20:30")}, - {Time: parseTimeUTC("2014-09-09 10:20:30")}, - {Time: parseTimeUTC("2014-09-10 10:20:30")}, - {Time: parseTimeUTC("2014-09-11 10:20:30")}, - {Time: parseTimeUTC("2014-09-20 10:20:30")}, - {Time: parseTimeUTC("2014-09-22 10:20:30")}, - {Time: parseTimeUTC("2014-08-08 10:20:30")}, - {Time: parseTimeUTC("2014-08-10 10:20:30")}, - {Time: parseTimeUTC("2014-08-12 10:20:30")}, - {Time: parseTimeUTC("2014-08-13 10:20:30")}, - {Time: parseTimeUTC("2014-08-13 10:20:30.1")}, - {Time: parseTimeUTC("2014-08-15 10:20:30")}, - {Time: parseTimeUTC("2014-08-18 10:20:30")}, - {Time: parseTimeUTC("2014-08-20 10:20:30")}, - {Time: parseTimeUTC("2014-08-21 10:20:30")}, - {Time: parseTimeUTC("2014-08-22 10:20:30")}, - {Time: parseTimeUTC("2014-10-01 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-10-02 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-10-05 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-10-06 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-10-08 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-10-09 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-10-10 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-10-11 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-10-20 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-10-22 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-11-08 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-11-10 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-11-12 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-11-13 10:20:30"), Tags: []string{"foo"}}, - {Time: parseTimeUTC("2014-11-13 10:20:30.1"), Tags: []string{"bar"}}, - {Time: parseTimeUTC("2014-11-15 10:20:30"), Tags: []string{"foo", "bar"}}, - {Time: parseTimeUTC("2014-11-18 10:20:30")}, - {Time: parseTimeUTC("2014-11-20 10:20:30")}, - {Time: parseTimeUTC("2014-11-21 10:20:30")}, - {Time: parseTimeUTC("2014-11-22 10:20:30")}, - {Time: parseTimeUTC("2015-09-01 10:20:30")}, - {Time: parseTimeUTC("2015-09-02 10:20:30")}, - {Time: parseTimeUTC("2015-09-05 10:20:30")}, - {Time: parseTimeUTC("2015-09-06 10:20:30")}, - {Time: parseTimeUTC("2015-09-08 10:20:30")}, - {Time: parseTimeUTC("2015-09-09 10:20:30")}, - {Time: parseTimeUTC("2015-09-10 10:20:30")}, - {Time: parseTimeUTC("2015-09-11 10:20:30")}, - {Time: parseTimeUTC("2015-09-20 10:20:30")}, - {Time: parseTimeUTC("2015-09-22 10:20:30")}, - {Time: parseTimeUTC("2015-08-08 10:20:30")}, - {Time: parseTimeUTC("2015-08-10 10:20:30")}, - {Time: parseTimeUTC("2015-08-12 10:20:30")}, - {Time: parseTimeUTC("2015-08-13 10:20:30")}, - {Time: parseTimeUTC("2015-08-13 10:20:30.1")}, - {Time: parseTimeUTC("2015-08-15 10:20:30")}, - {Time: parseTimeUTC("2015-08-18 10:20:30")}, - {Time: parseTimeUTC("2015-08-20 10:20:30")}, - {Time: parseTimeUTC("2015-08-21 10:20:30")}, - {Time: parseTimeUTC("2015-08-22 10:20:30")}, - {Time: parseTimeUTC("2015-10-01 10:20:30")}, - {Time: parseTimeUTC("2015-10-02 10:20:30")}, - {Time: parseTimeUTC("2015-10-05 10:20:30")}, - {Time: parseTimeUTC("2015-10-06 10:20:30")}, - {Time: parseTimeUTC("2015-10-08 10:20:30")}, - {Time: parseTimeUTC("2015-10-09 10:20:30")}, - {Time: parseTimeUTC("2015-10-10 10:20:30")}, - {Time: parseTimeUTC("2015-10-11 10:20:30")}, - {Time: parseTimeUTC("2015-10-20 10:20:30")}, - {Time: parseTimeUTC("2015-10-22 10:20:30")}, - {Time: parseTimeUTC("2015-10-22 10:20:30")}, - {Time: parseTimeUTC("2015-10-22 10:20:30"), Tags: []string{"foo", "bar"}}, - {Time: parseTimeUTC("2015-10-22 10:20:30"), Tags: []string{"foo", "bar"}}, - {Time: parseTimeUTC("2015-10-22 10:20:30"), Tags: []string{"foo", "bar"}, Paths: []string{"path1", "path2"}}, - {Time: parseTimeUTC("2015-11-08 10:20:30")}, - {Time: parseTimeUTC("2015-11-10 10:20:30")}, - {Time: parseTimeUTC("2015-11-12 10:20:30")}, - {Time: parseTimeUTC("2015-11-13 10:20:30")}, - {Time: parseTimeUTC("2015-11-13 10:20:30.1")}, - {Time: parseTimeUTC("2015-11-15 10:20:30")}, - {Time: parseTimeUTC("2015-11-18 10:20:30")}, - {Time: parseTimeUTC("2015-11-20 10:20:30")}, - {Time: parseTimeUTC("2015-11-21 10:20:30")}, - {Time: parseTimeUTC("2015-11-22 10:20:30")}, - {Time: parseTimeUTC("2016-01-01 01:02:03")}, - {Time: parseTimeUTC("2016-01-01 01:03:03")}, - {Time: parseTimeUTC("2016-01-01 07:08:03")}, - {Time: parseTimeUTC("2016-01-03 07:02:03")}, - {Time: parseTimeUTC("2016-01-04 10:23:03")}, - {Time: parseTimeUTC("2016-01-04 11:23:03")}, - {Time: parseTimeUTC("2016-01-04 12:23:03")}, - {Time: parseTimeUTC("2016-01-04 12:24:03")}, - {Time: parseTimeUTC("2016-01-04 12:28:03")}, - {Time: parseTimeUTC("2016-01-04 12:30:03")}, - {Time: parseTimeUTC("2016-01-04 16:23:03")}, - {Time: parseTimeUTC("2016-01-05 09:02:03")}, - {Time: parseTimeUTC("2016-01-06 08:02:03")}, - {Time: parseTimeUTC("2016-01-07 10:02:03")}, - {Time: parseTimeUTC("2016-01-08 20:02:03")}, - {Time: parseTimeUTC("2016-01-09 21:02:03")}, - {Time: parseTimeUTC("2016-01-12 21:02:03")}, - {Time: parseTimeUTC("2016-01-12 21:08:03")}, - {Time: parseTimeUTC("2016-01-18 12:02:03")}, -} - -var expireTests = []restic.ExpirePolicy{ - {}, - {Last: 10}, - {Last: 15}, - {Last: 99}, - {Last: 200}, - {Hourly: 20}, - {Daily: 3}, - {Daily: 10}, - {Daily: 30}, - {Last: 5, Daily: 5}, - {Last: 2, Daily: 10}, - {Weekly: 2}, - {Weekly: 4}, - {Daily: 3, Weekly: 4}, - {Monthly: 6}, - {Daily: 2, Weekly: 2, Monthly: 6}, - {Yearly: 10}, - {Daily: 7, Weekly: 2, Monthly: 3, Yearly: 10}, - {Tags: []restic.TagList{{"foo"}}}, - {Tags: []restic.TagList{{"foo", "bar"}}}, - {Tags: []restic.TagList{{"foo"}, {"bar"}}}, -} - func TestApplyPolicy(t *testing.T) { - for i, p := range expireTests { - keep, remove := restic.ApplyPolicy(testExpireSnapshots, p) + var testExpireSnapshots = restic.Snapshots{ + {Time: parseTimeUTC("2014-09-01 10:20:30")}, + {Time: parseTimeUTC("2014-09-02 10:20:30")}, + {Time: parseTimeUTC("2014-09-05 10:20:30")}, + {Time: parseTimeUTC("2014-09-06 10:20:30")}, + {Time: parseTimeUTC("2014-09-08 10:20:30")}, + {Time: parseTimeUTC("2014-09-09 10:20:30")}, + {Time: parseTimeUTC("2014-09-10 10:20:30")}, + {Time: parseTimeUTC("2014-09-11 10:20:30")}, + {Time: parseTimeUTC("2014-09-20 10:20:30")}, + {Time: parseTimeUTC("2014-09-22 10:20:30")}, + {Time: parseTimeUTC("2014-08-08 10:20:30")}, + {Time: parseTimeUTC("2014-08-10 10:20:30")}, + {Time: parseTimeUTC("2014-08-12 10:20:30")}, + {Time: parseTimeUTC("2014-08-13 10:20:30")}, + {Time: parseTimeUTC("2014-08-13 10:20:30.1")}, + {Time: parseTimeUTC("2014-08-15 10:20:30")}, + {Time: parseTimeUTC("2014-08-18 10:20:30")}, + {Time: parseTimeUTC("2014-08-20 10:20:30")}, + {Time: parseTimeUTC("2014-08-21 10:20:30")}, + {Time: parseTimeUTC("2014-08-22 10:20:30")}, + {Time: parseTimeUTC("2014-10-01 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-10-02 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-10-05 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-10-06 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-10-08 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-10-09 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-10-10 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-10-11 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-10-20 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-10-22 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-11-08 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-11-10 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-11-12 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-11-13 10:20:30"), Tags: []string{"foo"}}, + {Time: parseTimeUTC("2014-11-13 10:20:30.1"), Tags: []string{"bar"}}, + {Time: parseTimeUTC("2014-11-15 10:20:30"), Tags: []string{"foo", "bar"}}, + {Time: parseTimeUTC("2014-11-18 10:20:30")}, + {Time: parseTimeUTC("2014-11-20 10:20:30")}, + {Time: parseTimeUTC("2014-11-21 10:20:30")}, + {Time: parseTimeUTC("2014-11-22 10:20:30")}, + {Time: parseTimeUTC("2015-09-01 10:20:30")}, + {Time: parseTimeUTC("2015-09-02 10:20:30")}, + {Time: parseTimeUTC("2015-09-05 10:20:30")}, + {Time: parseTimeUTC("2015-09-06 10:20:30")}, + {Time: parseTimeUTC("2015-09-08 10:20:30")}, + {Time: parseTimeUTC("2015-09-09 10:20:30")}, + {Time: parseTimeUTC("2015-09-10 10:20:30")}, + {Time: parseTimeUTC("2015-09-11 10:20:30")}, + {Time: parseTimeUTC("2015-09-20 10:20:30")}, + {Time: parseTimeUTC("2015-09-22 10:20:30")}, + {Time: parseTimeUTC("2015-08-08 10:20:30")}, + {Time: parseTimeUTC("2015-08-10 10:20:30")}, + {Time: parseTimeUTC("2015-08-12 10:20:30")}, + {Time: parseTimeUTC("2015-08-13 10:20:30")}, + {Time: parseTimeUTC("2015-08-13 10:20:30.1")}, + {Time: parseTimeUTC("2015-08-15 10:20:30")}, + {Time: parseTimeUTC("2015-08-18 10:20:30")}, + {Time: parseTimeUTC("2015-08-20 10:20:30")}, + {Time: parseTimeUTC("2015-08-21 10:20:30")}, + {Time: parseTimeUTC("2015-08-22 10:20:30")}, + {Time: parseTimeUTC("2015-10-01 10:20:30")}, + {Time: parseTimeUTC("2015-10-02 10:20:30")}, + {Time: parseTimeUTC("2015-10-05 10:20:30")}, + {Time: parseTimeUTC("2015-10-06 10:20:30")}, + {Time: parseTimeUTC("2015-10-08 10:20:30")}, + {Time: parseTimeUTC("2015-10-09 10:20:30")}, + {Time: parseTimeUTC("2015-10-10 10:20:30")}, + {Time: parseTimeUTC("2015-10-11 10:20:30")}, + {Time: parseTimeUTC("2015-10-20 10:20:30")}, + {Time: parseTimeUTC("2015-10-22 10:20:30")}, + {Time: parseTimeUTC("2015-10-22 10:20:30")}, + {Time: parseTimeUTC("2015-10-22 10:20:30"), Tags: []string{"foo", "bar"}}, + {Time: parseTimeUTC("2015-10-22 10:20:30"), Tags: []string{"foo", "bar"}}, + {Time: parseTimeUTC("2015-10-22 10:20:30"), Tags: []string{"foo", "bar"}, Paths: []string{"path1", "path2"}}, + {Time: parseTimeUTC("2015-11-08 10:20:30")}, + {Time: parseTimeUTC("2015-11-10 10:20:30")}, + {Time: parseTimeUTC("2015-11-12 10:20:30")}, + {Time: parseTimeUTC("2015-11-13 10:20:30")}, + {Time: parseTimeUTC("2015-11-13 10:20:30.1")}, + {Time: parseTimeUTC("2015-11-15 10:20:30")}, + {Time: parseTimeUTC("2015-11-18 10:20:30")}, + {Time: parseTimeUTC("2015-11-20 10:20:30")}, + {Time: parseTimeUTC("2015-11-21 10:20:30")}, + {Time: parseTimeUTC("2015-11-22 10:20:30")}, + {Time: parseTimeUTC("2016-01-01 01:02:03")}, + {Time: parseTimeUTC("2016-01-01 01:03:03")}, + {Time: parseTimeUTC("2016-01-01 07:08:03")}, + {Time: parseTimeUTC("2016-01-03 07:02:03")}, + {Time: parseTimeUTC("2016-01-04 10:23:03")}, + {Time: parseTimeUTC("2016-01-04 11:23:03")}, + {Time: parseTimeUTC("2016-01-04 12:23:03")}, + {Time: parseTimeUTC("2016-01-04 12:24:03")}, + {Time: parseTimeUTC("2016-01-04 12:28:03")}, + {Time: parseTimeUTC("2016-01-04 12:30:03")}, + {Time: parseTimeUTC("2016-01-04 16:23:03")}, + {Time: parseTimeUTC("2016-01-05 09:02:03")}, + {Time: parseTimeUTC("2016-01-06 08:02:03")}, + {Time: parseTimeUTC("2016-01-07 10:02:03")}, + {Time: parseTimeUTC("2016-01-08 20:02:03")}, + {Time: parseTimeUTC("2016-01-09 21:02:03")}, + {Time: parseTimeUTC("2016-01-12 21:02:03")}, + {Time: parseTimeUTC("2016-01-12 21:08:03")}, + {Time: parseTimeUTC("2016-01-18 12:02:03")}, + } - t.Logf("test %d: returned keep %v, remove %v (of %v) expired snapshots for policy %v", - i, len(keep), len(remove), len(testExpireSnapshots), p) + var tests = []restic.ExpirePolicy{ + {}, + {Last: 10}, + {Last: 15}, + {Last: 99}, + {Last: 200}, + {Hourly: 20}, + {Daily: 3}, + {Daily: 10}, + {Daily: 30}, + {Last: 5, Daily: 5}, + {Last: 2, Daily: 10}, + {Weekly: 2}, + {Weekly: 4}, + {Daily: 3, Weekly: 4}, + {Monthly: 6}, + {Daily: 2, Weekly: 2, Monthly: 6}, + {Yearly: 10}, + {Daily: 7, Weekly: 2, Monthly: 3, Yearly: 10}, + {Tags: []restic.TagList{{"foo"}}}, + {Tags: []restic.TagList{{"foo", "bar"}}}, + {Tags: []restic.TagList{{"foo"}, {"bar"}}}, + {Within: parseDuration("1d")}, + {Within: parseDuration("2d")}, + {Within: parseDuration("7d")}, + {Within: parseDuration("1m")}, + {Within: parseDuration("1m14d")}, + {Within: parseDuration("1y1d1m")}, + } - if len(keep)+len(remove) != len(testExpireSnapshots) { - t.Errorf("test %d: len(keep)+len(remove) = %d != len(testExpireSnapshots) = %d", - i, len(keep)+len(remove), len(testExpireSnapshots)) - } + for i, p := range tests { + t.Run("", func(t *testing.T) { + keep, remove := restic.ApplyPolicy(testExpireSnapshots, p) - if p.Sum() > 0 && len(keep) > p.Sum() { - t.Errorf("not enough snapshots removed: policy allows %v snapshots to remain, but ended up with %v", - p.Sum(), len(keep)) - } + t.Logf("returned keep %v, remove %v (of %v) expired snapshots for policy %v", + len(keep), len(remove), len(testExpireSnapshots), p) - for _, sn := range keep { - t.Logf("test %d: keep snapshot at %v %s\n", i, sn.Time, sn.Tags) - } - for _, sn := range remove { - t.Logf("test %d: forget snapshot at %v %s\n", i, sn.Time, sn.Tags) - } + if len(keep)+len(remove) != len(testExpireSnapshots) { + t.Errorf("len(keep)+len(remove) = %d != len(testExpireSnapshots) = %d", + len(keep)+len(remove), len(testExpireSnapshots)) + } - goldenFilename := filepath.Join("testdata", fmt.Sprintf("policy_keep_snapshots_%d", i)) + if p.Sum() > 0 && len(keep) > p.Sum() { + t.Errorf("not enough snapshots removed: policy allows %v snapshots to remain, but ended up with %v", + p.Sum(), len(keep)) + } - if *updateGoldenFiles { - buf, err := json.MarshalIndent(keep, "", " ") + for _, sn := range keep { + t.Logf(" keep snapshot at %v %s", sn.Time, sn.Tags) + } + for _, sn := range remove { + t.Logf(" forget snapshot at %v %s", sn.Time, sn.Tags) + } + + goldenFilename := filepath.Join("testdata", fmt.Sprintf("policy_keep_snapshots_%d", i)) + + if *updateGoldenFiles { + buf, err := json.MarshalIndent(keep, "", " ") + if err != nil { + t.Fatalf("error marshaling result: %v", err) + } + + if err = ioutil.WriteFile(goldenFilename, buf, 0644); err != nil { + t.Fatalf("unable to update golden file: %v", err) + } + } + + buf, err := ioutil.ReadFile(goldenFilename) if err != nil { - t.Fatalf("error marshaling result: %v", err) + t.Fatalf("error loading golden file %v: %v", goldenFilename, err) } - if err = ioutil.WriteFile(goldenFilename, buf, 0644); err != nil { - t.Fatalf("unable to update golden file: %v", err) + var want restic.Snapshots + err = json.Unmarshal(buf, &want) + if err != nil { + t.Fatalf("error unmarshalling golden file %v: %v", goldenFilename, err) } - } - buf, err := ioutil.ReadFile(goldenFilename) - if err != nil { - t.Errorf("error loading golden file %v: %v", goldenFilename, err) - continue - } - - var want restic.Snapshots - err = json.Unmarshal(buf, &want) - if err != nil { - t.Errorf("error unmarshalling golden file %v: %v", goldenFilename, err) - continue - } - - if !reflect.DeepEqual(keep, want) { - t.Errorf("test %v: wrong result, want:\n %v\ngot:\n %v", i, want, keep) - continue - } + if !reflect.DeepEqual(keep, want) { + t.Fatalf("wrong result, want:\n %v\ngot:\n %v", want, keep) + } + }) } } diff --git a/internal/restic/testdata/policy_keep_snapshots_21 b/internal/restic/testdata/policy_keep_snapshots_21 new file mode 100644 index 000000000..319c9ab1c --- /dev/null +++ b/internal/restic/testdata/policy_keep_snapshots_21 @@ -0,0 +1,7 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/internal/restic/testdata/policy_keep_snapshots_22 b/internal/restic/testdata/policy_keep_snapshots_22 new file mode 100644 index 000000000..319c9ab1c --- /dev/null +++ b/internal/restic/testdata/policy_keep_snapshots_22 @@ -0,0 +1,7 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/internal/restic/testdata/policy_keep_snapshots_23 b/internal/restic/testdata/policy_keep_snapshots_23 new file mode 100644 index 000000000..667fb8b6d --- /dev/null +++ b/internal/restic/testdata/policy_keep_snapshots_23 @@ -0,0 +1,17 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/internal/restic/testdata/policy_keep_snapshots_24 b/internal/restic/testdata/policy_keep_snapshots_24 new file mode 100644 index 000000000..11be139f5 --- /dev/null +++ b/internal/restic/testdata/policy_keep_snapshots_24 @@ -0,0 +1,97 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:02:03Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/internal/restic/testdata/policy_keep_snapshots_25 b/internal/restic/testdata/policy_keep_snapshots_25 new file mode 100644 index 000000000..11be139f5 --- /dev/null +++ b/internal/restic/testdata/policy_keep_snapshots_25 @@ -0,0 +1,97 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:02:03Z", + "tree": null, + "paths": null + } +] \ No newline at end of file diff --git a/internal/restic/testdata/policy_keep_snapshots_26 b/internal/restic/testdata/policy_keep_snapshots_26 new file mode 100644 index 000000000..f36ec66b7 --- /dev/null +++ b/internal/restic/testdata/policy_keep_snapshots_26 @@ -0,0 +1,332 @@ +[ + { + "time": "2016-01-18T12:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-12T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-09T21:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-08T20:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-07T10:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-06T08:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-05T09:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T16:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:30:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:28:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:24:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T12:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T11:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-04T10:23:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-03T07:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T07:08:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:03:03Z", + "tree": null, + "paths": null + }, + { + "time": "2016-01-01T01:02:03Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-11-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": [ + "path1", + "path2" + ], + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-22T10:20:30Z", + "tree": null, + "paths": null, + "tags": [ + "foo", + "bar" + ] + }, + { + "time": "2015-10-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-10-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-11T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-09T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-08T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-06T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-05T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-02T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-09-01T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-22T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-21T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-20T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-18T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-15T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-13T10:20:30.1Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-13T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-12T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-10T10:20:30Z", + "tree": null, + "paths": null + }, + { + "time": "2015-08-08T10:20:30Z", + "tree": null, + "paths": null + } +] \ No newline at end of file