initial 2 commit

This commit is contained in:
Niccolo Borgioli 2023-07-09 17:49:33 +02:00
parent 732f728ff8
commit 7c11ba18fd
No known key found for this signature in database
GPG Key ID: D93C615F75EE4F0B
34 changed files with 1222 additions and 0 deletions

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
node_modules
/autorestic

BIN
bun.lockb Executable file

Binary file not shown.

14
migration.md Normal file
View File

@ -0,0 +1,14 @@
# Rename backend to repository
# Env variables
AUTORESTIC_BB_B2_ACCOUNT_ID=123 -> AUTORESTIC_BACKENDS_BB_ENV_B2**ACCOUNT**ID=123
- All fields can be configured by env now
- To escape `_` replace it with double underscore `__`
# Rest property on backend config
No rest property anymore, can be used in string extrapolation
# Every string is now replaceable with env variables

23
package.json Normal file
View File

@ -0,0 +1,23 @@
{
"private": true,
"name": "autorestic",
"module": "src/index.ts",
"type": "module",
"scripts": {
"schema:gen": "bun run ./scripts/generateSchema.ts",
"bin": "bun build ./src/index.ts --compile --outfile autorestic"
},
"devDependencies": {
"bun-types": "^0.6.0",
"typescript": "^5.0.0",
"zod-to-json-schema": "^3.21.2"
},
"dependencies": {
"@commander-js/extra-typings": "^11.0.0",
"commander": "^11.0.0",
"pino": "^8.14.1",
"pino-pretty": "^10.0.0",
"yaml": "^2.3.1",
"zod": "^3.21.4"
}
}

252
schema/config.json Normal file
View File

@ -0,0 +1,252 @@
{
"$ref": "#/definitions/mySchema",
"definitions": {
"mySchema": {
"type": "object",
"properties": {
"version": {
"type": "number",
"description": "version number"
},
"repos": {
"type": "object",
"additionalProperties": {
"type": "object",
"properties": {
"type": {
"type": "string",
"enum": [
"local",
"sftp",
"rest",
"swift",
"s3",
"b2",
"azure",
"gs",
"rclone"
],
"description": "type of repository"
},
"path": {
"type": "string",
"minLength": 1,
"description": "restic path"
},
"key": {
"$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/path",
"description": "encryption key for the repository"
},
"env": {
"type": "object",
"additionalProperties": {
"$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/path",
"description": "value of the environment variable"
},
"description": "environment variables"
},
"options": {
"type": "object",
"properties": {
"all": {
"type": "object",
"additionalProperties": {
"anyOf": [
{
"type": "boolean",
"const": true,
"description": "boolean flag"
},
{
"anyOf": [
{
"$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/path",
"description": "non-empty string that can extrapolate env variables inside it"
},
{
"type": "array",
"items": {
"$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/options/properties/all/additionalProperties/anyOf/1/anyOf/0"
},
"minItems": 1
}
]
}
],
"description": "value of option"
}
},
"backup": {
"$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/options/properties/all"
},
"forget": {
"$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/options/properties/all"
}
},
"additionalProperties": false,
"description": "options"
}
},
"required": [
"type",
"path",
"key"
],
"additionalProperties": false
},
"description": "available repositories"
},
"locations": {
"type": "object",
"additionalProperties": {
"type": "object",
"properties": {
"from": {
"anyOf": [
{
"$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/path",
"description": "local path to backup"
},
{
"type": "array",
"items": {
"$ref": "#/definitions/mySchema/properties/locations/additionalProperties/properties/from/anyOf/0"
},
"minItems": 1
}
]
},
"to": {
"anyOf": [
{
"$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/path",
"description": "repository to backup to"
},
{
"type": "array",
"items": {
"$ref": "#/definitions/mySchema/properties/locations/additionalProperties/properties/to/anyOf/0"
},
"minItems": 1
}
]
},
"copy": {
"type": "object",
"additionalProperties": {
"anyOf": [
{
"$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/path",
"description": "destination repository"
},
{
"type": "array",
"items": {
"$ref": "#/definitions/mySchema/properties/locations/additionalProperties/properties/copy/additionalProperties/anyOf/0"
},
"minItems": 1
}
]
}
},
"cron": {
"$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/path",
"description": "execute backups for the given cron job"
},
"hooks": {
"type": "object",
"properties": {
"before": {
"anyOf": [
{
"$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/path",
"description": "command to be executed"
},
{
"type": "array",
"items": {
"$ref": "#/definitions/mySchema/properties/locations/additionalProperties/properties/hooks/properties/before/anyOf/0"
},
"minItems": 1
}
],
"description": "list of commands"
},
"after": {
"$ref": "#/definitions/mySchema/properties/locations/additionalProperties/properties/hooks/properties/before",
"description": "list of commands"
},
"failure": {
"$ref": "#/definitions/mySchema/properties/locations/additionalProperties/properties/hooks/properties/before",
"description": "list of commands"
},
"success": {
"$ref": "#/definitions/mySchema/properties/locations/additionalProperties/properties/hooks/properties/before",
"description": "list of commands"
}
},
"additionalProperties": false,
"description": "hooks to be executed"
},
"options": {
"type": "object",
"properties": {
"all": {
"$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/options/properties/all"
},
"backup": {
"$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/options/properties/backup"
},
"forget": {
"$ref": "#/definitions/mySchema/properties/repos/additionalProperties/properties/options/properties/forget"
}
},
"additionalProperties": false,
"description": "native restic options"
},
"forget": {
"anyOf": [
{
"type": "boolean",
"description": "automatically run \"forget\" when backing up"
},
{
"type": "string",
"const": "prune",
"description": "also prune when forgetting"
}
]
}
},
"required": [
"from",
"to"
],
"additionalProperties": false,
"description": "Location"
},
"description": "available locations"
},
"global": {
"type": "object",
"properties": {
"options": {
"$ref": "#/definitions/mySchema/properties/locations/additionalProperties/properties/options",
"description": "native restic options"
}
},
"additionalProperties": false,
"description": "global configuration"
},
"extras": {}
},
"required": [
"version",
"repos",
"locations"
],
"additionalProperties": false
}
},
"$schema": "http://json-schema.org/draft-07/schema#"
}

17
scripts/generateSchema.ts Normal file
View File

@ -0,0 +1,17 @@
import { mkdir, rm, writeFile } from 'node:fs/promises'
import { zodToJsonSchema } from 'zod-to-json-schema'
import { ConfigSchema } from '../src/config/schema/config'
const OUTPUT = './schema'
await rm(OUTPUT, { recursive: true, force: true })
await mkdir(OUTPUT, { recursive: true })
const Schemas = {
config: ConfigSchema,
}
for (const [name, schema] of Object.entries(Schemas)) {
const jsonSchema = zodToJsonSchema(schema, 'mySchema')
await writeFile(`${OUTPUT}/${name}.json`, JSON.stringify(jsonSchema, null, 2), { encoding: 'utf-8' })
}

14
src/cmd/backup.ts Normal file
View File

@ -0,0 +1,14 @@
import { Log } from '../logger'
import { Context } from '../models/context'
export async function backup(ctx: Context) {
const log = Log.child({ cmd: 'check' })
log.trace('starting')
// Locations
for (const location of ctx.locations) {
await location.backup()
}
log.trace('done')
}

25
src/cmd/check.ts Normal file
View File

@ -0,0 +1,25 @@
import { unlockRepo, waitForRepo } from '../lock'
import { Log } from '../logger'
import { Context } from '../models/context'
import { isResticAvailable } from '../restic'
export async function check(ctx: Context) {
const l = Log.child({ cmd: 'check' })
l.trace('starting')
// Restic
isResticAvailable()
// Repos
for (const repo of ctx.repos) {
await waitForRepo(ctx, repo.name)
try {
await repo.init()
await repo.check()
} finally {
unlockRepo(ctx, repo.name)
}
}
l.trace('done')
}

21
src/config/env/file.test.ts vendored Normal file
View File

@ -0,0 +1,21 @@
import { describe, expect, test } from 'bun:test'
import { InvalidEnvFileLine } from '../../errors'
import { parseFile } from './file'
describe('env file', () => {
test('simple', () => {
expect(parseFile(`test_foo=ok`)).toEqual({ test_foo: 'ok' })
})
test('multiple values', () => {
expect(parseFile(`test_foo=ok\n \n spacing = foo \n`)).toEqual({ test_foo: 'ok', spacing: 'foo' })
})
test('invalid: key', () => {
expect(() => parseFile(`a=123\na f=ok`)).toThrow(new InvalidEnvFileLine('a f=ok'))
})
test('invalid: missing =', () => {
expect(() => parseFile(`a=123\na ok`)).toThrow(new InvalidEnvFileLine('a ok'))
})
})

55
src/config/env/file.ts vendored Normal file
View File

@ -0,0 +1,55 @@
import { exists, readFile } from 'node:fs/promises'
import { InvalidEnvFileLine } from '../../errors'
import { setByPath } from '../../utils/path'
import { relativePath } from '../resolution'
export function parseFile(contents: string) {
const variables: Record<string, string> = {}
const lines = contents
.trim()
.split('\n')
.map((l) => l.trim())
const matcher = /^\s*(?<variable>\w+)\s*=(?<value>.*)$/
for (const line of lines) {
if (!line) continue
const match = matcher.exec(line)
if (!match) throw new InvalidEnvFileLine(line)
variables[match.groups!.variable] = match.groups!.value.trim()
}
return variables
}
const PREFIX = 'AUTORESTIC_'
function envVariableToObjectPath(env: string): string {
if (env.startsWith(PREFIX)) env = env.replace(PREFIX, '')
return (
env
// Convert to object path
.replaceAll('_', '.')
// Escape the double unterscore. __ -> .. -> _
.replaceAll('..', '_')
.toLowerCase()
)
}
/**
* Fill the config file with the env file variables.
* These take precedence before the config file itself.
*/
export async function enrichConfig(rawConfig: any, path: string) {
const envFilePath = relativePath(path, '.autorestic.env')
let variables: Record<string, string> = {}
if (await exists(envFilePath)) {
const envFile = parseFile(await readFile(envFilePath, 'utf-8'))
Object.assign(variables, envFile)
}
Object.assign(variables, process.env)
for (const [key, value] of Object.entries(variables)) {
if (!key.startsWith(PREFIX)) continue
setByPath(rawConfig, envVariableToObjectPath(key), value)
}
}

29
src/config/index.ts Normal file
View File

@ -0,0 +1,29 @@
import { exists, readFile } from 'node:fs/promises'
import yaml from 'yaml'
import { ConfigFileNotFound, CustomError, InvalidConfigFile } from '../errors'
import { enrichConfig } from './env/file'
import { autoLocateConfig } from './resolution'
import { Config, ConfigWithMetaSchema } from './schema/config'
import { basename } from 'node:path'
export async function loadConfig(customPath?: string): Promise<Config> {
let path: string
if (customPath) {
path = customPath
if (!(await exists(path))) throw new ConfigFileNotFound([path])
} else {
path = await autoLocateConfig()
}
const rawConfig = await readFile(path, 'utf-8')
const config = yaml.parse(rawConfig)
await enrichConfig(config, path)
config.meta = { path: basename(path) }
const parsed = ConfigWithMetaSchema.safeParse(config)
if (!parsed.success)
throw new InvalidConfigFile(parsed.error.errors.map((e) => `${e.path.join(' > ')}: ${e.message}`))
// Check for semantics
return parsed.data
}

23
src/config/resolution.ts Normal file
View File

@ -0,0 +1,23 @@
import { exists } from 'node:fs/promises'
import { dirname, isAbsolute, join, resolve } from 'node:path'
import { ConfigFileNotFound } from '../errors'
const DEFAULT_DIRS = ['./', '~/', '~/.config/autorestic']
const FILENAMES = ['.autorestic.yaml', '.autorestic.yml', '.autorestic.json']
export async function autoLocateConfig(): Promise<string> {
const paths = DEFAULT_DIRS
const xdgHome = process.env['XDG_CONFIG_HOME']
if (xdgHome) paths.push(xdgHome)
for (const path in paths) {
for (const filename in FILENAMES) {
const file = join(path, filename)
if (await exists(file)) return file
}
}
throw new ConfigFileNotFound(paths)
}
export function relativePath(base: string, path: string): string {
return isAbsolute(path) ? path : resolve(base, path)
}

View File

@ -0,0 +1,20 @@
import { ZodTypeAny, z } from 'zod'
import { Log } from '../../logger'
export const NonEmptyString = z
.string()
.min(1)
// Extrapolate env variables from a string
.transform((s) => {
return s.replaceAll(/\$(\w+)|\${(\w+)}/g, (_, g0, g1) => {
const variable = g0 || g1
const value = process.env[variable] ?? ''
if (!value) Log.error(`cannot find environment variable "${variable}" to replace in ${s}`)
return value
})
})
.describe('non-empty string that can extrapolate env variables inside it')
export function OptionallyArray<T extends ZodTypeAny>(type: T) {
return z.union([type, z.array(type).min(1)])
}

View File

@ -0,0 +1,69 @@
import { z } from 'zod'
import { asArray } from '../../utils/array'
import { RepositorySchema } from './repository'
import { NonEmptyString } from './common'
import { LocationSchema } from './location'
import { OptionsSchema } from './options'
export const ConfigSchema = z.strictObject({
version: z.number().describe('version number'),
repos: z.record(NonEmptyString.describe('repository name'), RepositorySchema).describe('available repositories'),
locations: z.record(NonEmptyString.describe('location name'), LocationSchema).describe('available locations'),
global: z
.strictObject({
options: OptionsSchema.optional(),
})
.describe('global configuration')
.optional(),
extras: z.any().optional(),
})
const ConfigMeta = z
.strictObject({
path: NonEmptyString.describe('The path of the loaded config'),
})
.describe('Meta information about the config')
export const ConfigWithMetaSchema = ConfigSchema.extend({
meta: ConfigMeta,
}).superRefine((config, ctx) => {
const availableRepos = Object.keys(config.repos)
for (const [name, location] of Object.entries(config.locations)) {
const locationPath = [...ctx.path, 'locations', name]
const toRepos = asArray(location.to)
// Check if all target repos are valid
for (const to of toRepos) {
if (!availableRepos.includes(to)) {
const message = `location "${name}" has an invalid repository "${to}"`
ctx.addIssue({ message, code: 'custom', path: [...locationPath, 'to'] })
}
}
// Check copy field
if (!location.copy) continue
for (const [source, destinations] of Object.entries(location.copy)) {
const path = [...locationPath, 'copy', source]
if (!toRepos.includes(source))
ctx.addIssue({
code: 'custom',
path,
message: `copy source "${source}" must be also a backup target`,
})
for (const destination of asArray(destinations)) {
if (destination === source)
ctx.addIssue({
code: 'custom',
path: [...path, destination],
message: `destination repository "${destination}" cannot be also the source in copy field`,
})
if (!availableRepos.includes(destination))
ctx.addIssue({
code: 'custom',
path: [...path, destination],
message: `destination repository "${destination}" does not exist`,
})
}
}
}
})
export type Config = z.infer<typeof ConfigWithMetaSchema>

View File

@ -0,0 +1,14 @@
import { z } from 'zod'
import { NonEmptyString, OptionallyArray } from './common'
const Command = NonEmptyString.describe('command to be executed')
const Commands = OptionallyArray(Command).describe('list of commands')
export const HooksSchema = z
.strictObject({
before: Commands.optional(),
after: Commands.optional(),
failure: Commands.optional(),
success: Commands.optional(),
})
.describe('hooks to be executed')

View File

@ -0,0 +1,28 @@
import { z } from 'zod'
import { NonEmptyString, OptionallyArray } from './common'
import { HooksSchema } from './hooks'
import { OptionsSchema } from './options'
export const LocationSchema = z
.strictObject({
from: OptionallyArray(NonEmptyString.describe('local path to backup')),
to: OptionallyArray(NonEmptyString.describe('repository to backup to')),
copy: z
.record(
NonEmptyString.describe('source repository from which to copy from'),
OptionallyArray(NonEmptyString.describe('destination repository'))
)
.optional(),
// adapter:
cron: NonEmptyString.describe('execute backups for the given cron job').optional(),
hooks: HooksSchema.optional(),
options: OptionsSchema.optional(),
forget: z
.union([
z.boolean().describe('automatically run "forget" when backing up'),
z.literal('prune').describe('also prune when forgetting'),
])
.optional(),
})
.describe('Location')

View File

@ -0,0 +1,15 @@
import { z } from 'zod'
import { NonEmptyString, OptionallyArray } from './common'
const OptionSchema = z.record(
NonEmptyString.describe('native restic option'),
z.union([z.literal(true).describe('boolean flag'), OptionallyArray(NonEmptyString)]).describe('value of option')
)
export const OptionsSchema = z
.strictObject({
all: OptionSchema.optional(),
backup: OptionSchema.optional(),
forget: OptionSchema.optional(),
})
.describe('native restic options')

View File

@ -0,0 +1,18 @@
import { z } from 'zod'
import { NonEmptyString } from './common'
import { OptionsSchema } from './options'
export const RepositorySchema = z.strictObject({
type: z.enum(['local', 'sftp', 'rest', 'swift', 's3', 'b2', 'azure', 'gs', 'rclone']).describe('type of repository'),
path: NonEmptyString.describe('restic path'),
key: NonEmptyString.describe('encryption key for the repository'),
env: z
.record(
NonEmptyString.describe('environment variable'),
NonEmptyString.describe('value of the environment variable')
)
.transform((env) => Object.fromEntries(Object.entries(env).map(([key, value]) => [key.toUpperCase(), value])))
.describe('environment variables')
.optional(),
options: OptionsSchema.describe('options').optional(),
})

47
src/errors/index.ts Normal file
View File

@ -0,0 +1,47 @@
function formatLines(lines: string[]) {
return lines.map((p) => `${p}`).join('\n')
}
export class CustomError extends Error {}
export class InvalidEnvFileLine extends CustomError {
constructor(line: string) {
super(`invalid env file line: "${line}"`)
}
}
export class NotImplemented extends CustomError {
constructor(functionality: string) {
super(`not implemented: ${functionality}`)
}
}
export class ConfigFileNotFound extends CustomError {
constructor(paths: string[]) {
super(`could not locate config file.\nthe following paths were tried:\n${formatLines(paths)}`)
}
}
export class InvalidConfigFile extends CustomError {
constructor(errors: string[]) {
super(`could not parse the config file.\n${formatLines(errors)}`)
}
}
export class BinaryNotAvailable extends CustomError {
constructor(binary: string) {
super(`binary "${binary}" is not available in $PATH`)
}
}
export class ResticError extends CustomError {
constructor(errors: string[]) {
super(`internal restic error.\n${formatLines(errors)}`)
}
}
export class LockfileAlreadyLocked extends CustomError {
constructor(repo: string) {
super(`cannot acquire lock for repository "${repo}", already in use`)
}
}

138
src/index.ts Normal file
View File

@ -0,0 +1,138 @@
import { Command, Help, Option, program } from '@commander-js/extra-typings'
import { loadConfig } from './config'
import { CustomError, NotImplemented } from './errors'
import { Log, LogLevel, setLevelFromFlag } from './logger'
import { check } from './cmd/check'
import { Context } from './models/context'
import { backup } from './cmd/backup'
export const helpConfig: Partial<Help> = {
showGlobalOptions: true,
sortOptions: true,
sortSubcommands: true,
helpWidth: 1,
}
program
.name('autorestic')
.description('configuration manager and runner for restic')
.version('2.0.0-alpha.0')
.configureHelp(helpConfig)
.allowExcessArguments(false)
.allowUnknownOption(false)
// Global options
program.option('-c, --config <file>', 'specify custom configuration file')
program.option('-v', 'verbosity', (_, previous) => previous + 1, 1)
program.addOption(new Option('--ci', 'CI mode').env('CI').default(false))
// Common Options
const specificLocation = new Option('-l, --location <locations...>', 'location name, multiple possible')
specificLocation.variadic = true
const allLocations = new Option('-a, --all', 'all locations')
const specificRepo = new Option('-r, --repository <names...>', 'repository name, multiple possible')
specificLocation.variadic = true
const allRepos = new Option('-a, --all', 'all repositories')
function mergeOptions<T extends {}>(local: T, p: Command) {
const globals = p.optsWithGlobals() as { config?: string; verbosity: number; ci: boolean }
return {
...globals,
...local,
}
}
program.hook('preAction', (command) => {
// @ts-ignore
const v: number = command.opts().v
setLevelFromFlag(v)
})
program
.command('check')
.description('check if the config is valid and sets up the repositories')
.configureHelp(helpConfig)
.action(async (options, p) => {
const merged = mergeOptions(options, p)
const config = await loadConfig(merged.config)
const ctx = new Context(config)
await check(ctx)
})
program
.command('backup')
.description('create backups')
.configureHelp(helpConfig)
.addOption(specificLocation)
.addOption(allLocations)
.action(async (options, p) => {
// throw new NotImplemented('backup')
const merged = mergeOptions(options, p)
const config = await loadConfig(merged.config)
const ctx = new Context(config)
await backup(ctx)
})
program
.command('exec')
.description('execute arbitrary native restic commands for given repositories')
.configureHelp(helpConfig)
.addOption(specificRepo)
.addOption(allRepos)
.allowExcessArguments(true)
.action((options, p) => {
throw new NotImplemented('exec')
})
program
.command('forget')
.description('forget snapshots according to the specified policies')
.configureHelp(helpConfig)
.addOption(specificLocation)
.addOption(allLocations)
// Pass natively
// .option('--dry-run', 'do not write changes, show what would be affected')
// .option('--prune', 'also prune repository')
.action((options) => {
throw new NotImplemented('backup')
})
program
.command('restore')
.description('restore a snapshot to a given location')
.option('--force', 'overwrite target folder')
.option('--from <repository>', 'repository from which to restore')
.option('--to <path>', 'path where to restore the data')
.option('-l, --location <location>', 'location to be restored')
.argument('[snapshot-id]', 'snapshot to be restored. if empty latest will be taken')
.action(() => {
throw new NotImplemented('restore')
})
const self = new Command('self').description('utility commands for managing autorestic').configureHelp(helpConfig)
self.command('install').action(() => {
throw new NotImplemented('install')
})
self.command('uninstall').action(() => {
throw new NotImplemented('uninstall')
})
self.command('upgrade').action(() => {
throw new NotImplemented('upgrade')
})
self.command('completion').action(() => {
throw new NotImplemented('completion')
})
program.addCommand(self)
try {
await program.parseAsync()
} catch (e) {
if (e instanceof CustomError) {
Log.fatal(e.message)
} else if (e instanceof Error) {
Log.fatal(`unknown error: ${e.message}`)
}
process.exit(1)
} finally {
// TODO: Unlock
}

20
src/lock/index.test.ts Normal file
View File

@ -0,0 +1,20 @@
import { describe, expect, mock, test, beforeEach } from 'bun:test'
import { lockRepo } from '.'
import { Context } from '../models/context'
import { mkdir, rm } from 'node:fs/promises'
const mockPath = './test/'
const mockContext: Context = { config: { meta: { path: mockPath } } } as any
describe('lock', () => {
beforeEach(async () => {
// Cleanup lock file
await rm(mockPath, { recursive: true, force: true })
await mkdir(mockPath, { recursive: true })
})
test('lock', () => {
lockRepo(mockContext, 'foo')
// lockRepo(mockContext, 'foo')
})
})

78
src/lock/index.ts Normal file
View File

@ -0,0 +1,78 @@
import { readFileSync, writeFileSync } from 'node:fs'
import yaml from 'yaml'
import { relativePath } from '../config/resolution'
import { LockfileAlreadyLocked } from '../errors'
import { Log } from '../logger'
import { Context } from '../models/context'
import { Lockfile, LockfileSchema } from './schema'
import { wait } from '../utils/time'
const LOCKFILE = '.autorestic.lock'
const VERSION = 2
const l = Log.child({ command: 'lock' })
function load(ctx: Context): Lockfile {
const defaultLockfile = { version: VERSION, cron: {}, running: {} }
try {
const path = relativePath(ctx.config.meta.path, LOCKFILE)
l.trace('looking for lock file', { path })
// throw new Error(path)
const rawConfig = readFileSync(path, 'utf-8')
const config = yaml.parse(rawConfig)
const parsed = LockfileSchema.safeParse(config)
if (!parsed.success) return defaultLockfile
if (parsed.data.version < VERSION) {
l.debug('lockfile is old and will be overwritten')
return defaultLockfile
}
return parsed.data
} catch {
return defaultLockfile
}
}
function write(ctx: Context, lockfile: Lockfile) {
const path = relativePath(ctx.config.meta.path, LOCKFILE)
writeFileSync(path, yaml.stringify(lockfile), 'utf-8')
}
export function lockRepo(ctx: Context, repo: string) {
const lock = load(ctx)
l.trace('trying to lock repository', { repo })
if (lock.running[repo]) throw new LockfileAlreadyLocked(repo)
lock.running[repo] = true
write(ctx, lock)
}
/**
* Waits for a repo to become unlocked, and errors if it does not succeed in the given timeout.
*
* @param [timeout=10] max seconds to wait for repo to become unlocked
*/
export async function waitForRepo(ctx: Context, repo: string, timeout = 10) {
const now = Date.now()
while (Date.now() - now < timeout * 1_000) {
try {
lockRepo(ctx, repo)
l.trace('repo is free again', { repo })
break
} catch {
l.trace('waiting for repo to be unlocked', { repo })
await wait(0.1) // Wait for 100ms
}
}
throw new LockfileAlreadyLocked(repo)
}
export function updateLastRun(ctx: Context, location: string) {
const lock = load(ctx)
lock.cron[location] = Date.now()
write(ctx, lock)
}
export function unlockRepo(ctx: Context, repo: string) {
l.trace('unlocking repository', { repo })
const lock = load(ctx)
lock.running[repo] = false
write(ctx, lock)
}

13
src/lock/schema.ts Normal file
View File

@ -0,0 +1,13 @@
import { z } from 'zod'
export const LockfileSchema = z.strictObject({
version: z.number().min(0).describe('lockfile version'),
running: z
.record(z.string().describe('repository'), z.boolean().describe('whether repository is running'))
.describe('running information for each repository'),
cron: z
.record(z.string().describe('location'), z.number().describe('timestamp of last backup'))
.describe('information about last run for a given location. in milliseconds'),
})
export type Lockfile = z.infer<typeof LockfileSchema>

37
src/logger.ts Normal file
View File

@ -0,0 +1,37 @@
import Pino, { type LoggerOptions } from 'pino'
import Pretty from 'pino-pretty'
// https://getpino.io/#/docs/api?id=loggerlevel-string-gettersetter
export enum LogLevel {
Trace = 'trace',
Debug = 'debug',
Info = 'info',
Warn = 'warn',
Error = 'error',
Fatal = 'fatal',
Silent = 'silent',
}
const pretty = !process.env.CI
const options: LoggerOptions = {
base: undefined,
level: LogLevel.Info,
}
export const Log = pretty ? Pino(options, Pretty({ colorize: true })) : Pino(options)
export function setLevelFromFlag(flag: number) {
switch (flag) {
case 1:
Log.level = LogLevel.Info
break
case 2:
Log.level = LogLevel.Debug
break
case 3:
Log.level = LogLevel.Trace
break
default:
Log.error('invalid logging level')
}
}

25
src/models/context.ts Normal file
View File

@ -0,0 +1,25 @@
import { Config } from '../config/schema/config'
import { CustomError } from '../errors'
import { Location } from './location'
import { Repository } from './repository'
export class Context {
repos: Repository[]
locations: Location[]
constructor(public config: Config) {
this.repos = Object.entries(config.repos).map(([name, r]) => new Repository(this, name, r))
this.locations = Object.entries(config.locations).map(([name, l]) => new Location(this, name, l))
}
getRepo(name: string) {
const repo = this.repos.find((r) => r.name === name)
if (!repo) throw new CustomError(`could not find backend "${name}"`)
return repo
}
getLocation(name: string) {
const location = this.locations.find((l) => l.name === name)
if (!location) throw new CustomError(`could not find location "${name}"`)
return location
}
}

28
src/models/location.ts Normal file
View File

@ -0,0 +1,28 @@
import { Logger } from 'pino'
import { z } from 'zod'
import { LocationSchema } from '../config/schema/location'
import { Log } from '../logger'
import { asArray } from '../utils/array'
import { Context } from './context'
import { execute } from '../restic'
export class Location {
l: Logger
constructor(public ctx: Context, public name: string, public data: z.infer<typeof LocationSchema>) {
this.l = Log.child({ location: name })
}
async backup() {
this.l.trace('backing up location')
for (const name of asArray(this.data.to)) {
const repo = this.ctx.getRepo(name)
this.l.debug(repo.name)
await execute({
command: 'restic',
args: ['backup', '--dry-run'],
env: repo.env,
})
}
}
}

66
src/models/repository.ts Normal file
View File

@ -0,0 +1,66 @@
import { Logger } from 'pino'
import { z } from 'zod'
import { relativePath } from '../config/resolution'
import { Config } from '../config/schema/config'
import { RepositorySchema } from '../config/schema/repository'
import { ResticError } from '../errors'
import { Log } from '../logger'
import { execute } from '../restic'
import { Context } from './context'
export class Repository {
l: Logger
constructor(public ctx: Context, public name: string, public data: z.infer<typeof RepositorySchema>) {
this.l = Log.child({ repository: this.name })
}
get repository(): string {
switch (this.data.type) {
case 'local':
return relativePath(this.ctx.config.meta.path, this.data.path)
case 'b2':
case 'azure':
case 'gs':
case 's3':
case 'sftp':
case 'rclone':
case 'swift':
case 'rest':
return `${this.data.type}:${this.data.path}`
break
}
}
get env() {
return {
...this.data.env,
RESTIC_PASSWORD: this.data.key,
RESTIC_REPOSITORY: this.repository,
}
}
/**
* true if initialized
* false if already initialized
*/
async init(): Promise<boolean> {
this.l.trace('initializing')
const output = await execute({ command: 'restic', args: ['init'], env: this.env })
if (!output.ok) {
if (output.stderr.includes('config file already exists')) {
this.l.debug('already initialized')
return false
}
throw new ResticError([output.stderr])
}
this.l.debug('initialized repository')
return true
}
async check() {
this.l.trace('checking')
const output = await execute({ command: 'restic', args: ['check'], env: this.env })
if (!output.ok) throw new ResticError(['could not check repository', output.stderr])
}
}

44
src/restic/index.ts Normal file
View File

@ -0,0 +1,44 @@
import { execFile } from 'node:child_process'
import { Log } from '../logger'
import { BinaryNotAvailable } from '../errors'
export type ExecutionContext = {
command: string
args?: string[]
env?: Record<string, string>
}
export async function execute({
env,
args,
command,
}: ExecutionContext): Promise<{ code: number; stderr: string; stdout: string; ok: boolean }> {
return new Promise((resolve) => {
execFile(command, args ?? [], { env }, (err, stdout, stderr) => {
const code = err?.code ?? 0
resolve({
code,
ok: code === 0,
stderr,
stdout,
})
})
})
}
export async function isBinaryAvailable(command: string): Promise<boolean> {
const l = Log.child({ command })
try {
l.trace('checking if command is installed')
const result = await execute({ command })
return result.ok
} catch {
l.trace('not installed')
return false
}
}
export async function isResticAvailable() {
const bin = 'restic'
const installed = await isBinaryAvailable(bin)
if (!installed) throw new BinaryNotAvailable(bin)
}

12
src/utils/array.test.ts Normal file
View File

@ -0,0 +1,12 @@
import { describe, expect, test } from 'bun:test'
import { isSubset } from './array'
describe('set theory', () => {
test('subset', () => {
expect(isSubset([1], [1, 2])).toBe(true)
expect(isSubset([1], [2])).toBe(false)
expect(isSubset([], [])).toBe(true)
expect(isSubset([1, 2, 3], [1, 2])).toBe(false)
expect(isSubset([1, 2], [1, 2])).toBe(true)
})
})

7
src/utils/array.ts Normal file
View File

@ -0,0 +1,7 @@
export function asArray<T>(singleOrArray: T | T[]): T[] {
return Array.isArray(singleOrArray) ? singleOrArray : [singleOrArray]
}
export function isSubset<T>(subset: T[], set: T[]): boolean {
return subset.every((v) => set.includes(v))
}

33
src/utils/path.test.ts Normal file
View File

@ -0,0 +1,33 @@
import { expect, test, describe } from 'bun:test'
import { setByPath } from './path'
describe('set by path', () => {
test('simple', () => {
expect(setByPath({}, 'a', true)).toEqual({ a: true })
expect(setByPath({}, 'f', { ok: true })).toEqual({ f: { ok: true } })
expect(setByPath([], '0', true)).toEqual([true])
expect(setByPath([], '2', false)).toEqual([undefined, undefined, false])
})
test('object', () => {
expect(setByPath({}, 'a.b', true)).toEqual({ a: { b: true } })
expect(setByPath({}, 'a.b.c', true)).toEqual({ a: { b: { c: true } } })
expect(setByPath({ a: true }, 'b', false)).toEqual({ a: true, b: false })
expect(setByPath({ a: { b: true } }, 'a.c', false)).toEqual({ a: { b: true, c: false } })
expect(() => setByPath({ a: 'foo' }, 'a.b', true)).toThrow()
expect(setByPath({ a: 'foo' }, 'a', true)).toEqual({ a: true })
})
test('array', () => {
expect(() => setByPath([], 'a', true)).toThrow()
expect(setByPath([], '0', true)).toEqual([true])
expect(setByPath([], '0.0.0', true)).toEqual([[[true]]])
expect(setByPath([], '0.1.2', true)).toEqual([[undefined, [undefined, undefined, true]]])
})
test('mixed', () => {
expect(setByPath({ items: [] }, 'items.0.name', 'John')).toEqual({ items: [{ name: 'John' }] })
expect(setByPath([], '0.name', 'John')).toEqual([{ name: 'John' }])
})
})

24
src/utils/path.ts Normal file
View File

@ -0,0 +1,24 @@
function parseKey(key: any) {
const asNumber = parseInt(key)
const isString = isNaN(asNumber)
return [isString ? key : asNumber, isString]
}
export function setByPath(source: object, path: string, value: unknown) {
const segments = path.split('.')
const last = segments.length - 1
let node: any = source
for (const [i, segment] of segments.entries()) {
const [key, isString] = parseKey(segment)
if (Array.isArray(node) && isString) throw new Error(`array require a numeric index`)
if (typeof node !== 'object') throw new Error(`could not set path "${segment}" on ${node}.`)
if (i === last) {
node[key] = value
} else {
const [_, isNextString] = parseKey(segments[i + 1])
if (node[key] === undefined) node[key] = isNextString ? {} : []
node = node[key]
}
}
return source
}

3
src/utils/time.ts Normal file
View File

@ -0,0 +1,3 @@
export function wait(seconds: number): Promise<never> {
return new Promise((resolve) => setTimeout(resolve, seconds * 1000))
}

8
tsconfig.json Normal file
View File

@ -0,0 +1,8 @@
{
"extends": "./node_modules/bun-types/tsconfig.json",
"compilerOptions": {
"strict": true,
"noFallthroughCasesInSwitch": true,
"noEmit": true
}
}