Merge pull request #125 from cupcakearmy/1.5.0

1.5.0
This commit is contained in:
Nicco 2021-11-20 17:11:44 +01:00 committed by GitHub
commit e055e28cfe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 337 additions and 109 deletions

6
.dockerignore Normal file
View File

@ -0,0 +1,6 @@
*
!**/*.go
!build
!cmd
!internal
!go.*

View File

@ -3,16 +3,44 @@ name: Main
on:
push:
tags:
- 'v*.*.*'
- "v*.*.*"
jobs:
build:
docker:
runs-on: ubuntu-latest
steps:
- name: Set up QEMU
uses: docker/setup-qemu-action@v1
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
- name: Docker Labels
id: meta
uses: crazy-max/ghaction-docker-meta@v2
with:
images: cupcakearmy/autorestic
tags: |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=semver,pattern={{major}}
- name: Login to DockerHub
uses: docker/login-action@v1
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build and push
uses: docker/build-push-action@v2
with:
platforms: linux/amd64,linux/arm64
push: true
tags: ${{ steps.meta.outputs.tags }}
release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- uses: actions/setup-go@v2
with:
go-version: '^1.16.3'
go-version: "^1.16.3"
- name: Build
run: go run build/build.go
@ -25,4 +53,4 @@ jobs:
with:
files: dist/*
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

View File

@ -5,9 +5,27 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [1.5.0] - 2021-11-20
### Added
- Support for multiple paths
- Improved error handling
- Allow for specific snapshot to be restored
- Docker image
### Fixed
- rclone in docker volumes
### Changed
- [Breaking Change] Declaration of docker volumes. See: https://autorestic.vercel.app/migration/1.4_1.5
- [Breaking Change] Hooks default executing directory now defaults to the config file directory. See: https://autorestic.vercel.app/migration/1.4_1.5
## [1.4.1] - 2021-10-31
### Fixes
### Fixed
- Numeric values from config files not being passed to env.

12
Dockerfile Normal file
View File

@ -0,0 +1,12 @@
FROM golang:1.16-alpine as builder
WORKDIR /app
COPY go.* .
RUN go mod download
COPY . .
RUN go build
FROM alpine
RUN apk add --no-cache restic rclone
COPY --from=builder /app/autorestic /usr/bin/autorestic
CMD [ "autorestic" ]

View File

@ -30,7 +30,7 @@ var backupCmd = &cobra.Command{
}
location, _ := internal.GetLocation(splitted[0])
errs := location.Backup(false, specificBackend)
for err := range errs {
for _, err := range errs {
colors.Error.Println(err)
errors++
}

View File

@ -9,8 +9,9 @@ import (
)
var restoreCmd = &cobra.Command{
Use: "restore",
Use: "restore [snapshot id]",
Short: "Restore backup for location",
Args: cobra.MaximumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
err := lock.Lock()
CheckErr(err)
@ -24,7 +25,11 @@ var restoreCmd = &cobra.Command{
target, _ := cmd.Flags().GetString("to")
from, _ := cmd.Flags().GetString("from")
force, _ := cmd.Flags().GetBool("force")
err = l.Restore(target, from, force)
snapshot := ""
if len(args) > 0 {
snapshot = args[0]
}
err = l.Restore(target, from, force, snapshot)
CheckErr(err)
},
}

View File

@ -40,6 +40,11 @@
> [Uninstall](/cli/uninstall)
> [Upgrade](/cli/upgrade)
> :Collapse label=Migration
>
> [0.x → 1.0](/migration/0.x_1.0)
> [1.4 → 1.5](/migration/1.4_1.5)
[Examples](/examples)
[QA](/qa)
[Community](/community)

View File

@ -1,12 +1,12 @@
# Restore
```bash
autorestic restore [-l, --location] [--from backend] [--to <out dir>] [-f, --force]
autorestic restore [-l, --location] [--from backend] [--to <out dir>] [-f, --force] [snapshot]
```
This will restore all the locations to the selected target. If for one location there are more than one backends specified autorestic will take the first one.
This will restore the location to the selected target. If for one location there are more than one backends specified autorestic will take the first one. If no specific snapshot is specified `autorestic` will use `latest`.
The `--to` path has to be empty as no data will be overwritten by default. If you are sure you can pass the `-f, --force` flag and the data will be overwritten in the destination. However note that this will overwrite all the data existent in the backup, not only the 1 file that is missing e.g.
If you are sure you can pass the `-f, --force` flag and the data will be overwritten in the destination. However note that this will overwrite all the data existent in the backup, not only the 1 file that is missing e.g.
## Example

View File

@ -10,6 +10,10 @@ wget -qO - https://raw.githubusercontent.com/CupCakeArmy/autorestic/master/insta
## Alternatives
### Docker
There is an official docker image over at [cupcakearmy/autorestic](https://hub.docker.com/r/cupcakearmy/autorestic).
### Manual
You can download the right binary from the release page and simply copy it to `/usr/local/bin` or whatever path you prefer. Autoupdates will still work.
@ -20,6 +24,6 @@ If you are on macOS you can install through brew: `brew install autorestic`.
### AUR
If you are on Arch there is an [AUR Package](https://aur.archlinux.org/packages/autorestic-bin/) (looking for maintainers).
~~If you are on Arch there is an [AUR Package](https://aur.archlinux.org/packages/autorestic-bin/) (looking for maintainers).~~ - Deprecated
> :ToCPrevNext

View File

@ -3,7 +3,7 @@
autorestic supports docker volumes directly, without needing them to be mounted to the host filesystem.
```yaml | docker-compose.yml
version: '3.7'
version: '3.8'
volumes:
data:
@ -18,13 +18,9 @@ services:
```yaml | .autorestic.yml
locations:
- name: hello
from: volume:my-data
to:
- remote
backends:
- name: remote
foo:
from: my-data
type: volume
# ...
```

View File

@ -26,7 +26,7 @@ locations:
## Example
In this example, whenever `autorestic` runs `restic backup` it will append a `--tag abc --tag` to the native command.
In this example, whenever `autorestic` runs `restic backup` it will append a `--tag foo --tag bar` to the native command.
```yaml
locations:
@ -40,6 +40,12 @@ locations:
- bar
```
## Priority
Options can be set globally, on the backends or on the locations.
The priority is as follows: `location > backend > global`.
## Global Options
It is possible to specify global flags that will be run every time restic is invoked. To do so specify them under `global` in your config file.

View File

@ -1,6 +1,4 @@
# Upgrade
## From `0.x` to `1.0`
# From `0.x` to `1.0`
Most of the config file is remained compatible, however to clean up the backends custom environment variables were moved from the root object to an `env` object.

View File

@ -0,0 +1,60 @@
# Migration from `1.4` to `1.5`
## ⚠️ Important notes
The way snapshots are referenced in the `restore` and `prune` commands has been changed. Before they were referenced by the path. Now every backup is tagged and those tags are then referenced in the cli. This means that when running restore and forget commands old backups are not taken into account anymore.
## Config files
- The config file now required to have a version number. This has to be added with `version: 2` at the root.
- Hooks now optionally support `dir: /some/dir` in the [options object](https://pkg.go.dev/github.com/cupcakearmy/autorestic/internal#Hooks).
- Docker volumes don't get prefixed with `volume:` anymore, rather you have to set the `type: volume` in the [location config](https://pkg.go.dev/github.com/cupcakearmy/autorestic/internal#Hooks).
See detailed instructions below.
## Config Version
```yaml
version: 2 # Added
backends:
# ...
```
## Hooks
Since `1.5` multiple sources for a location are possible.
For this reason, while before hooks where executed in the folder of the source, now they are executed in the directory of the config `.autorestic.yaml`.
You can overwrite this behavior with the new `dir` option in the hook section of the config.
```yaml
locations:
l1:
# ...
from: /foo/bar
hooks:
dir: /foo/bar
before: pwd
```
## Docker volumes
The syntax with docker volumes has changed and needs to be adjusted.
```yaml
# Before
locations:
foo:
from: volume:my-data
```
```yaml
# After
locations:
foo:
from: my-data
type: volume
```
> :ToCPrevNext

View File

@ -0,0 +1,4 @@
# Migration
- [From 0.x to 1.0](/migration/0.x_1.0)
- [From 1.4 to 1.5](/migration/1.4_1.5)

View File

@ -157,30 +157,52 @@ func (b Backend) ExecDocker(l Location, args []string) (string, error) {
if err != nil {
return "", err
}
volume := l.getVolumeName()
path, _ := l.getPath()
volume := l.From[0]
options := ExecuteOptions{
Command: "docker",
Envs: env,
}
dir := "/data"
args = append([]string{"restic"}, args...)
docker := []string{
"run", "--rm",
"--pull", "always",
"--entrypoint", "ash",
"--workdir", path,
"--volume", volume + ":" + path,
"--workdir", dir,
"--volume", volume + ":" + dir,
}
// Use of docker host, not the container host
if hostname, err := os.Hostname(); err == nil {
docker = append(docker, "--hostname", hostname)
}
if b.Type == "local" {
switch b.Type {
case "local":
actual := env["RESTIC_REPOSITORY"]
docker = append(docker, "--volume", actual+":"+"/repo")
env["RESTIC_REPOSITORY"] = "/repo"
case "b2":
case "s3":
case "azure":
case "gs":
// No additional setup needed
case "rclone":
// Read host rclone config and mount it into the container
configFile, err := ExecuteCommand(ExecuteOptions{Command: "rclone"}, "config", "file")
if err != nil {
return "", err
}
splitted := strings.Split(strings.TrimSpace(configFile), "\n")
configFilePath := splitted[len(splitted)-1]
docker = append(docker, "--volume", configFilePath+":"+"/root/.config/rclone/rclone.conf:ro")
// Install rclone in the container
args = append([]string{"apk", "add", "rclone", "&&"}, args...)
default:
return "", fmt.Errorf("Backend type \"%s\" is not supported as volume endpoint", b.Type)
}
for key, value := range env {
docker = append(docker, "--env", key+"="+value)
}
docker = append(docker, "restic/restic", "-c", "restic "+strings.Join(args, " "))
docker = append(docker, "restic/restic", "-c", strings.Join(args, " "))
out, err := ExecuteCommand(options, docker...)
return out, err
}

View File

@ -16,7 +16,7 @@ import (
"github.com/spf13/viper"
)
const VERSION = "1.4.1"
const VERSION = "1.5.0"
var CI bool = false
var VERBOSE bool = false
@ -26,6 +26,7 @@ type OptionMap map[string][]interface{}
type Options map[string]OptionMap
type Config struct {
Version string `yaml:"version"`
Extras interface{} `yaml:"extras"`
Locations map[string]Location `yaml:"locations"`
Backends map[string]Backend `yaml:"backends"`
@ -35,7 +36,19 @@ type Config struct {
var once sync.Once
var config *Config
func exitConfig(err error, msg string) {
if err != nil {
colors.Error.Println(err)
}
if msg != "" {
colors.Error.Println(msg)
}
lock.Unlock()
os.Exit(1)
}
func GetConfig() *Config {
if config == nil {
once.Do(func() {
if err := viper.ReadInConfig(); err == nil {
@ -53,11 +66,24 @@ func GetConfig() *Config {
return
}
var versionConfig interface{}
viper.UnmarshalKey("version", &versionConfig)
if versionConfig == nil {
exitConfig(nil, "no version specified in config file. please see docs on how to migrate")
}
version, ok := versionConfig.(int)
if !ok {
exitConfig(nil, "version specified in config file is not an int")
} else {
// Check for version
if version != 2 {
exitConfig(nil, "unsupported config version number. please check the docs for migration\nhttps://autorestic.vercel.app/migration/")
}
}
config = &Config{}
if err := viper.UnmarshalExact(config); err != nil {
colors.Error.Println("Could not parse config file!")
lock.Unlock()
os.Exit(1)
exitConfig(err, "Could not parse config file!")
}
})
}
@ -81,7 +107,11 @@ func (c *Config) Describe() {
var tmp string
colors.PrimaryPrint(`Location: "%s"`, name)
colors.PrintDescription("From", l.From)
tmp = ""
for _, path := range l.From {
tmp += fmt.Sprintf("\t%s %s\n", colors.Success.Sprint("←"), path)
}
colors.PrintDescription("From", tmp)
tmp = ""
for _, to := range l.To {
@ -269,3 +299,15 @@ func getOptions(options Options, key string) []string {
}
return selected
}
func combineOptions(key string, l Location, b Backend) []string {
// Priority: location > backend > global
var options []string
gFlags := getOptions(GetConfig().Global, key)
bFlags := getOptions(b.Options, key)
lFlags := getOptions(l.Options, key)
options = append(options, gFlags...)
options = append(options, bFlags...)
options = append(options, lFlags...)
return options
}

View File

@ -17,14 +17,14 @@ import (
type LocationType string
const (
TypeLocal LocationType = "local"
TypeVolume LocationType = "volume"
VolumePrefix string = "volume:"
TypeLocal LocationType = "local"
TypeVolume LocationType = "volume"
)
type HookArray = []string
type Hooks struct {
Dir string `yaml:"dir"`
Before HookArray `yaml:"before,omitempty"`
After HookArray `yaml:"after,omitempty"`
Success HookArray `yaml:"success,omitempty"`
@ -33,7 +33,8 @@ type Hooks struct {
type Location struct {
name string `yaml:",omitempty"`
From string `yaml:"from,omitempty"`
From []string `yaml:"from,omitempty"`
Type string `yaml:"type,omitempty"`
To []string `yaml:"to,omitempty"`
Hooks Hooks `yaml:"hooks,omitempty"`
Cron string `yaml:"cron,omitempty"`
@ -47,21 +48,32 @@ func GetLocation(name string) (Location, bool) {
}
func (l Location) validate() error {
if l.From == "" {
if len(l.From) == 0 {
return fmt.Errorf(`Location "%s" is missing "from" key`, l.name)
}
if l.getType() == TypeLocal {
if from, err := GetPathRelativeToConfig(l.From); err != nil {
return err
} else {
if stat, err := os.Stat(from); err != nil {
t, err := l.getType()
if err != nil {
return err
}
switch t {
case TypeLocal:
for _, path := range l.From {
if from, err := GetPathRelativeToConfig(path); err != nil {
return err
} else {
if !stat.IsDir() {
return fmt.Errorf("\"%s\" is not valid directory for location \"%s\"", from, l.name)
if stat, err := os.Stat(from); err != nil {
return err
} else {
if !stat.IsDir() {
return fmt.Errorf("\"%s\" is not valid directory for location \"%s\"", from, l.name)
}
}
}
}
case TypeVolume:
if len(l.From) > 1 {
return fmt.Errorf(`location "%s" has more than one docker volume`, l.name)
}
}
if len(l.To) == 0 {
@ -77,10 +89,17 @@ func (l Location) validate() error {
return nil
}
func ExecuteHooks(commands []string, options ExecuteOptions) error {
func (l Location) ExecuteHooks(commands []string, options ExecuteOptions) error {
if len(commands) == 0 {
return nil
}
if l.Hooks.Dir != "" {
if dir, err := GetPathRelativeToConfig(l.Hooks.Dir); err != nil {
return err
} else {
options.Dir = dir
}
}
colors.Secondary.Println("\nRunning hooks")
for _, command := range commands {
colors.Body.Println("> " + command)
@ -97,39 +116,38 @@ func ExecuteHooks(commands []string, options ExecuteOptions) error {
return nil
}
func (l Location) getType() LocationType {
if strings.HasPrefix(l.From, VolumePrefix) {
return TypeVolume
func (l Location) getType() (LocationType, error) {
t := strings.ToLower(l.Type)
if t == "" || t == "local" {
return TypeLocal, nil
} else if t == "volume" {
return TypeVolume, nil
}
return TypeLocal
return "", fmt.Errorf("invalid location type \"%s\"", l.Type)
}
func (l Location) getVolumeName() string {
return strings.TrimPrefix(l.From, VolumePrefix)
func buildTag(parts ...string) string {
parts = append([]string{"ar"}, parts...)
return strings.Join(parts, ":")
}
func (l Location) getPath() (string, error) {
t := l.getType()
switch t {
case TypeLocal:
if path, err := GetPathRelativeToConfig(l.From); err != nil {
return "", err
} else {
return path, nil
}
case TypeVolume:
return "/volume/" + l.name + "/" + l.getVolumeName(), nil
}
return "", fmt.Errorf("could not get path for location \"%s\"", l.name)
func (l Location) getLocationTags() string {
return buildTag("location", l.name)
}
func (l Location) Backup(cron bool, specificBackend string) []error {
var errors []error
var backends []string
colors.PrimaryPrint(" Backing up location \"%s\" ", l.name)
t := l.getType()
t, err := l.getType()
if err != nil {
errors = append(errors, err)
return errors
}
cwd, _ := GetPathRelativeToConfig(".")
options := ExecuteOptions{
Command: "bash",
Dir: cwd,
Envs: map[string]string{
"AUTORESTIC_LOCATION": l.name,
},
@ -137,18 +155,11 @@ func (l Location) Backup(cron bool, specificBackend string) []error {
if err := l.validate(); err != nil {
errors = append(errors, err)
colors.Error.Print(err)
goto after
}
if t == TypeLocal {
dir, _ := GetPathRelativeToConfig(l.From)
colors.Faint.Printf("Executing under: \"%s\"\n", dir)
options.Dir = dir
}
// Hooks
if err := ExecuteHooks(l.Hooks.Before, options); err != nil {
if err := l.ExecuteHooks(l.Hooks.Before, options); err != nil {
errors = append(errors, err)
goto after
}
@ -173,30 +184,38 @@ func (l Location) Backup(cron bool, specificBackend string) []error {
continue
}
lFlags := getOptions(l.Options, "backup")
bFlags := getOptions(backend.Options, "backup")
cmd := []string{"backup"}
cmd = append(cmd, lFlags...)
cmd = append(cmd, bFlags...)
cmd = append(cmd, combineOptions("backup", l, backend)...)
if cron {
cmd = append(cmd, "--tag", "cron")
cmd = append(cmd, "--tag", buildTag("cron"))
}
cmd = append(cmd, ".")
cmd = append(cmd, "--tag", l.getLocationTags())
backupOptions := ExecuteOptions{
Dir: options.Dir,
Envs: env,
}
var out string
switch t {
case TypeLocal:
for _, from := range l.From {
path, err := GetPathRelativeToConfig(from)
if err != nil {
errors = append(errors, err)
goto after
}
cmd = append(cmd, path)
}
out, err = ExecuteResticCommand(backupOptions, cmd...)
case TypeVolume:
ok := CheckIfVolumeExists(l.From[0])
if !ok {
errors = append(errors, fmt.Errorf("volume \"%s\" does not exist", l.From[0]))
continue
}
cmd = append(cmd, "/data")
out, err = backend.ExecDocker(l, cmd)
}
if err != nil {
colors.Error.Println(out)
errors = append(errors, err)
continue
}
@ -213,7 +232,7 @@ func (l Location) Backup(cron bool, specificBackend string) []error {
}
// After hooks
if err := ExecuteHooks(l.Hooks.After, options); err != nil {
if err := l.ExecuteHooks(l.Hooks.After, options); err != nil {
errors = append(errors, err)
}
@ -224,22 +243,19 @@ after:
} else {
commands = l.Hooks.Success
}
if err := ExecuteHooks(commands, options); err != nil {
if err := l.ExecuteHooks(commands, options); err != nil {
errors = append(errors, err)
}
colors.Success.Println("Done")
if len(errors) == 0 {
colors.Success.Println("Done")
}
return errors
}
func (l Location) Forget(prune bool, dry bool) error {
colors.PrimaryPrint("Forgetting for location \"%s\"", l.name)
path, err := l.getPath()
if err != nil {
return err
}
for _, to := range l.To {
backend, _ := GetBackend(to)
colors.Secondary.Printf("For backend \"%s\"\n", backend.name)
@ -250,17 +266,14 @@ func (l Location) Forget(prune bool, dry bool) error {
options := ExecuteOptions{
Envs: env,
}
lFlags := getOptions(l.Options, "forget")
bFlags := getOptions(backend.Options, "forget")
cmd := []string{"forget", "--path", path}
cmd := []string{"forget", "--tag", l.getLocationTags()}
if prune {
cmd = append(cmd, "--prune")
}
if dry {
cmd = append(cmd, "--dry-run")
}
cmd = append(cmd, lFlags...)
cmd = append(cmd, bFlags...)
cmd = append(cmd, combineOptions("forget", l, backend)...)
out, err := ExecuteResticCommand(options, cmd...)
if VERBOSE {
colors.Faint.Println(out)
@ -282,7 +295,7 @@ func (l Location) hasBackend(backend string) bool {
return false
}
func (l Location) Restore(to, from string, force bool) error {
func (l Location) Restore(to, from string, force bool, snapshot string) error {
if from == "" {
from = l.To[0]
} else if !l.hasBackend(from) {
@ -293,16 +306,20 @@ func (l Location) Restore(to, from string, force bool) error {
if err != nil {
return err
}
colors.PrimaryPrint("Restoring location \"%s\"", l.name)
backend, _ := GetBackend(from)
path, err := l.getPath()
if err != nil {
return nil
if snapshot == "" {
snapshot = "latest"
}
colors.Secondary.Println("Restoring lastest snapshot")
colors.Body.Printf("%s → %s.\n", from, path)
switch l.getType() {
colors.PrimaryPrint("Restoring location \"%s\"", l.name)
backend, _ := GetBackend(from)
colors.Secondary.Printf("Restoring %s@%s → %s\n", snapshot, backend.name, to)
t, err := l.getType()
if err != nil {
return err
}
switch t {
case TypeLocal:
// Check if target is empty
if !force {
@ -322,9 +339,9 @@ func (l Location) Restore(to, from string, force bool) error {
}
}
}
err = backend.Exec([]string{"restore", "--target", to, "--path", path, "latest"})
err = backend.Exec([]string{"restore", "--target", to, "--tag", l.getLocationTags(), snapshot})
case TypeVolume:
_, err = backend.ExecDocker(l, []string{"restore", "--target", ".", "--path", path, "latest"})
_, err = backend.ExecDocker(l, []string{"restore", "--target", "/", "--tag", l.getLocationTags(), snapshot})
}
if err != nil {
return err

View File

@ -77,3 +77,8 @@ func CopyFile(from, to string) error {
}
return nil
}
func CheckIfVolumeExists(volume string) bool {
_, err := ExecuteCommand(ExecuteOptions{Command: "docker"}, "volume", "inspect", volume)
return err == nil
}