diff --git a/.travis.yml b/.travis.yml index 4da25ff8b..0eb90bbe2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,7 @@ install: script: - gox -verbose -os "${GOX_OS}" -tags "release" ./cmd/restic - gox -verbose -os "${GOX_OS}" -tags "debug" ./cmd/restic + - go run build.go - go run run_tests.go all.cov - GOARCH=386 RESTIC_TEST_INTEGRATION=0 go test ./... - goveralls -coverprofile=all.cov -service=travis-ci -repotoken "$COVERALLS_TOKEN" || true diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 48b8e5600..e31b5e135 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,6 +24,42 @@ If you are unsure what to do, please have a look at the issues, especially those tagged [minor complexity](https://github.com/restic/restic/labels/minor%20complexity). + +Development Environment +======================= + +For development, it is recommended to check out the restic repository within a +`GOPATH`, an introductory text is +["How to Write Go Code"](https://golang.org/doc/code.html). It is recommended +to have a working directory, we're using `~/work/restic` in the following. This +directory mainly contains the directory `src`, where the source code is stored. + +First, create the necessary directory structure and clone the restic repository +to the correct location: + + $ mkdir --parents ~/work/restic/src/github.com/restic + $ cd ~/work/restic/src/github.com/restic + $ git clone https://github.com/restic/restic + $ cd restic + +Now we're in the main directory of the restic repository. The last step is to +set the environment variable `$GOPATH` to the correct value: + + $ export GOPATH=~/work/restic:~/work/restic/src/github.com/restic/restic/Godeps/_workspace + +The following commands can be used to run all the tests: + + $ go test ./... + ok github.com/restic/restic 8.174s + [...] + +The restic binary can be built from the directory `cmd/restic` this way: + + $ cd cmd/restic + $ go build + $ ./restic version + restic compiled manually on go1.4.2 + Providing Patches ================= @@ -34,7 +70,10 @@ down to the following steps: 1. First we would kindly ask you to fork our project on GitHub if you haven't done so already. - 2. Clone the repository locally and create a new branch. + 2. Clone the repository locally and create a new branch. If you are working on + the code itself, please set up the development environment as described in + the previous section and instead of cloning add your fork on GitHub as a + remote to the clone of the restic repository. 3. Then commit your changes as fine grained as possible, as smaller patches, that handle one and only one issue are easier to discuss and merge. 4. Push the new branch with your changes to your fork of the repository. diff --git a/Makefile b/Makefile index 68419d5be..e5a11720b 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,4 @@ -.PHONY: all clean env test bench gox test-integration - -TMPGOPATH=$(PWD)/.gopath -VENDORPATH=$(PWD)/Godeps/_workspace -BASE=github.com/restic/restic -BASEPATH=$(TMPGOPATH)/src/$(BASE) - -GOPATH=$(TMPGOPATH):$(VENDORPATH) - -GOTESTFLAGS ?= -v -GOX_OS ?= linux darwin openbsd freebsd -SFTP_PATH ?= /usr/lib/ssh/sftp-server - -CMDS=$(patsubst cmd/%,%,$(wildcard cmd/*)) -CMDS_DEBUG=$(patsubst %,%.debug,$(CMDS)) +.PHONY: all clean test SOURCE=$(wildcard *.go) $(wildcard */*.go) $(wildcard */*/*.go) @@ -20,43 +6,17 @@ export GOPATH GOX_OS all: restic -.gopath: - mkdir -p .gopath/src/github.com/restic - ln -snf ../../../.. .gopath/src/github.com/restic/restic +restic: $(SOURCE) + go run build.go -%: cmd/% .gopath $(SOURCE) - cd $(BASEPATH) && \ - go build -a -tags release -ldflags "-s" -o $@ ./$< - -%.debug: cmd/% .gopath $(SOURCE) - cd $(BASEPATH) && \ - go build -a -tags debug -ldflags "-s" -o $@ ./$< +restic.debug: $(SOURCE) + go run build.go -tags debug clean: - rm -rf .gopath $(CMDS) $(CMDS_DEBUG) *.cov restic_* - go clean ./... + rm -rf restic restic.debug -test: .gopath - cd $(BASEPATH) && \ - go test $(GOTESTFLAGS) ./... +test: $(SOURCE) + go run run_tests.go /dev/null -bench: .gopath - cd $(BASEPATH) && \ - go test $(GOTESTFLAGS) -bench ./... - -gox: .gopath $(SOURCE) - cd $(BASEPATH) && \ - gox -verbose -os "$(GOX_OS)" ./cmd/restic - -all.cov: .gopath $(SOURCE) - cd $(BASEPATH) && go run run_tests.go all.cov - -env: - @echo export GOPATH=\"$(GOPATH)\" - -goenv: - go env - -list: .gopath - cd $(BASEPATH) && \ - go list ./... +all.cov: $(SOURCE) + go run run_tests.go all.cov diff --git a/README.md b/README.md index 063f7e009..d070aa64f 100644 --- a/README.md +++ b/README.md @@ -43,11 +43,10 @@ Restic is a program that does backups right. The design goals are: Building ======== -Install Go/Golang (at least version 1.3), then run `make`, afterwards you'll -find the binary in the current directory: +Install Go/Golang (at least version 1.3), then run `go run build.go`, +afterwards you'll find the binary in the current directory: - $ make - [...] + $ go run build.go $ ./restic --help Usage: @@ -73,7 +72,6 @@ find the binary in the current directory: snapshots show snapshots version display version - Contribute and Documentation ============================ @@ -82,6 +80,13 @@ Contributions are welcome! More information can be found in restic and the data structures stored on disc is contained in [`doc/Design.md`](doc/Design.md). +Development +=========== + +For development, please have a look at [`CONTRIBUTING.md`](CONTRIBUTING.md), +especially the section "Development Environment". If you have any questions, +please get in touch! + Contact ======= diff --git a/build.go b/build.go new file mode 100644 index 000000000..765432f0c --- /dev/null +++ b/build.go @@ -0,0 +1,279 @@ +// +build ignore + +package main + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + "time" +) + +var ( + verbose bool + keepGopath bool +) + +const timeFormat = "2006-01-02 15:04:05" + +// specialDir returns true if the file begins with a special character ('.' or '_'). +func specialDir(name string) bool { + if name == "." { + return false + } + + base := filepath.Base(name) + return base[0] == '_' || base[0] == '.' +} + +// updateGopath builds a valid GOPATH at dst, with all Go files in src/ copied +// to dst/prefix/, so calling +// +// updateGopath("/tmp/gopath", "/home/u/restic", "github.com/restic/restic") +// +// with "/home/u/restic" containing the file "foo.go" yields the following tree +// at "/tmp/gopath": +// +// /tmp/gopath +// └── src +// └── github.com +// └── restic +// └── restic +// └── foo.go +func updateGopath(dst, src, prefix string) error { + return filepath.Walk(src, func(name string, fi os.FileInfo, err error) error { + if specialDir(name) { + if fi.IsDir() { + return filepath.SkipDir + } + + return nil + } + + if fi.IsDir() { + return nil + } + + ext := path.Ext(name) + if ext != ".go" && ext != ".s" { + return nil + } + + intermediatePath, err := filepath.Rel(src, name) + if err != nil { + return err + } + + fileSrc := filepath.Join(src, intermediatePath) + fileDst := filepath.Join(dst, "src", prefix, intermediatePath) + + return copyFile(fileDst, fileSrc) + }) +} + +// copyFile creates dst from src, preserving file attributes and timestamps. +func copyFile(dst, src string) error { + fi, err := os.Stat(src) + if err != nil { + return err + } + + fsrc, err := os.Open(src) + if err != nil { + return err + } + + if err = os.MkdirAll(filepath.Dir(dst), 0755); err != nil { + fmt.Printf("MkdirAll(%v)\n", filepath.Dir(dst)) + return err + } + + fdst, err := os.Create(dst) + if err != nil { + return err + } + + if _, err = io.Copy(fdst, fsrc); err != nil { + return err + } + + if err == nil { + err = fsrc.Close() + } + + if err == nil { + err = fdst.Close() + } + + if err == nil { + err = os.Chmod(dst, fi.Mode()) + } + + if err == nil { + err = os.Chtimes(dst, fi.ModTime(), fi.ModTime()) + } + + return nil +} + +// die prints the message with fmt.Fprintf() to stderr and exits with an error +// code. +func die(message string, args ...interface{}) { + fmt.Fprintf(os.Stderr, message, args...) + os.Exit(1) +} + +func showUsage(output io.Writer) { + fmt.Fprintf(output, "USAGE: go run build.go OPTIONS\n") + fmt.Fprintf(output, "\n") + fmt.Fprintf(output, "OPTIONS:\n") + fmt.Fprintf(output, " -v --verbose output more messages\n") +} + +func verbosePrintf(message string, args ...interface{}) { + if !verbose { + return + } + + fmt.Printf(message, args...) +} + +// cleanEnv returns a clean environment with GOPATH and GOBIN removed (if +// present). +func cleanEnv() (env []string) { + for _, v := range os.Environ() { + if strings.HasPrefix(v, "GOPATH=") || strings.HasPrefix(v, "GOBIN=") { + continue + } + + env = append(env, v) + } + + return env +} + +// build runs "go build args..." with GOPATH set to gopath. +func build(gopath string, args ...string) error { + args = append([]string{"build"}, args...) + cmd := exec.Command("go", args...) + cmd.Env = append(cleanEnv(), "GOPATH="+gopath) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + verbosePrintf("go %s\n", args) + + return cmd.Run() +} + +// getVersion returns a version string, either from the file VERSION in the +// current directory or from git. +func getVersion() string { + v, err := ioutil.ReadFile("VERSION") + version := strings.TrimSpace(string(v)) + if err == nil { + verbosePrintf("version from file 'VERSION' is %s\n", version) + return version + } + + return gitVersion() +} + +// gitVersion returns a version string that identifies the currently checked +// out git commit. +func gitVersion() string { + cmd := exec.Command("git", "describe", + "--long", "--tags", "--dirty", "--always") + out, err := cmd.Output() + if err != nil { + die("git describe returned error: %v\n", err) + } + + version := strings.TrimSpace(string(out)) + verbosePrintf("git version is %s\n", version) + return version +} + +func main() { + buildTags := []string{} + + skipNext := false + params := os.Args[1:] + for i, arg := range params { + if skipNext { + skipNext = false + continue + } + + switch arg { + case "-v", "--verbose": + verbose = true + case "-k", "--keep-gopath": + keepGopath = true + case "-t", "-tags", "--tags": + skipNext = true + buildTags = strings.Split(params[i+1], " ") + case "-h": + showUsage(os.Stdout) + default: + fmt.Fprintf(os.Stderr, "Error: unknown option %q\n\n", arg) + showUsage(os.Stderr) + os.Exit(1) + } + } + + if len(buildTags) == 0 { + verbosePrintf("adding build-tag release\n") + buildTags = []string{"release"} + } + + for i := range buildTags { + buildTags[i] = strings.TrimSpace(buildTags[i]) + } + + verbosePrintf("build tags: %s\n", buildTags) + + root, err := os.Getwd() + if err != nil { + die("Getwd(): %v\n", err) + } + + gopath, err := ioutil.TempDir("", "restic-build-") + if err != nil { + die("TempDir(): %v\n", err) + } + + verbosePrintf("create GOPATH at %v\n", gopath) + if err = updateGopath(gopath, root, "github.com/restic/restic"); err != nil { + die("copying files from %v to %v failed: %v\n", root, gopath, err) + } + + vendor := filepath.Join(root, "Godeps", "_workspace", "src") + if err = updateGopath(gopath, vendor, ""); err != nil { + die("copying files from %v to %v failed: %v\n", root, gopath, err) + } + + version := getVersion() + compileTime := time.Now().Format(timeFormat) + args := []string{ + "-tags", strings.Join(buildTags, " "), + "-ldflags", fmt.Sprintf(`-s -X main.version %q -X main.compiledAt %q`, version, compileTime), + "-o", "restic", "github.com/restic/restic/cmd/restic", + } + err = build(gopath, args...) + if err != nil { + fmt.Fprintf(os.Stderr, "build failed: %v\n", err) + } + + if !keepGopath { + verbosePrintf("remove %v\n", gopath) + if err = os.RemoveAll(gopath); err != nil { + die("remove GOPATH at %s failed: %v\n", err) + } + } else { + fmt.Printf("leaving temporary GOPATH at %v\n", gopath) + } +} diff --git a/cmd/restic/Makefile b/cmd/restic/Makefile deleted file mode 100644 index c1d2ba9ff..000000000 --- a/cmd/restic/Makefile +++ /dev/null @@ -1,18 +0,0 @@ -.PHONY: all clean debug - -# include config file if it exists --include $(CURDIR)/config.mk - -all: restic - -debug: restic.debug - -restic: $(wildcard *.go) $(wildcard ../../*.go) $(wildcard ../../*/*.go) - go build -a - -restic.debug: $(wildcard *.go) $(wildcard ../../*.go) $(wildcard ../../*/*.go) - go build -a -tags debug -o restic.debug - -clean: - go clean - rm -f restic restic.debug diff --git a/cmd/restic/cmd_version.go b/cmd/restic/cmd_version.go index 966c7b635..5e64790a1 100644 --- a/cmd/restic/cmd_version.go +++ b/cmd/restic/cmd_version.go @@ -18,7 +18,8 @@ func init() { } func (cmd CmdVersion) Execute(args []string) error { - fmt.Printf("restic %s on %v\n", version, runtime.Version()) + fmt.Printf("restic %s\ncompiled at %s with %v\n", + version, compiledAt, runtime.Version()) return nil } diff --git a/cmd/restic/global.go b/cmd/restic/global.go index 4ea0a4e54..b71f7aa25 100644 --- a/cmd/restic/global.go +++ b/cmd/restic/global.go @@ -17,6 +17,7 @@ import ( ) var version = "compiled manually" +var compiledAt = "unknown time" type GlobalOptions struct { Repo string `short:"r" long:"repo" description:"Repository directory to backup to/restore from"` diff --git a/cmd/restic/version.sh b/cmd/restic/version.sh deleted file mode 100755 index 6abc6f8ee..000000000 --- a/cmd/restic/version.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - -VERSION=$(git log --max-count=1 --pretty='%ad-%h' --date=short HEAD 2>/dev/null) - -if [ -n "$VERSION" ]; then - if ! sh -c "git diff -s --exit-code && git diff --cached -s --exit-code"; then - VERSION+="+" - fi -else - VERSION="unknown version" -fi - -echo $VERSION