Merge pull request #197 from restic/new-build-system

Refactor build system for end users

.travis.yml

@@ -31,6 +31,7 @@ install:
 script:
   - gox -verbose -os "${GOX_OS}" -tags "release" ./cmd/restic
   - gox -verbose -os "${GOX_OS}" -tags "debug" ./cmd/restic
+  - go run build.go
   - go run run_tests.go all.cov
   - GOARCH=386 RESTIC_TEST_INTEGRATION=0 go test ./...
   - goveralls -coverprofile=all.cov -service=travis-ci -repotoken "$COVERALLS_TOKEN" || true

CONTRIBUTING.md

@@ -24,6 +24,42 @@ If you are unsure what to do, please have a look at the issues, especially
 those tagged
 [minor complexity](https://github.com/restic/restic/labels/minor%20complexity).
 
+
+Development Environment
+=======================
+
+For development, it is recommended to check out the restic repository within a
+`GOPATH`, an introductory text is
+["How to Write Go Code"](https://golang.org/doc/code.html). It is recommended
+to have a working directory, we're using `~/work/restic` in the following. This
+directory mainly contains the directory `src`, where the source code is stored.
+
+First, create the necessary directory structure and clone the restic repository
+to the correct location:
+
+    $ mkdir --parents ~/work/restic/src/github.com/restic
+    $ cd ~/work/restic/src/github.com/restic
+    $ git clone https://github.com/restic/restic
+    $ cd restic
+
+Now we're in the main directory of the restic repository. The last step is to
+set the environment variable `$GOPATH` to the correct value:
+
+    $ export GOPATH=~/work/restic:~/work/restic/src/github.com/restic/restic/Godeps/_workspace
+
+The following commands can be used to run all the tests:
+
+    $ go test ./...
+    ok          github.com/restic/restic        8.174s
+    [...]
+
+The restic binary can be built from the directory `cmd/restic` this way:
+
+    $ cd cmd/restic
+    $ go build
+    $ ./restic version
+    restic compiled manually on go1.4.2
+
 Providing Patches
 =================
 
@@ -34,7 +70,10 @@ down to the following steps:
 
  1. First we would kindly ask you to fork our project on GitHub if you haven't
     done so already.
- 2. Clone the repository locally and create a new branch.
+ 2. Clone the repository locally and create a new branch. If you are working on
+    the code itself, please set up the development environment as described in
+    the previous section and instead of cloning add your fork on GitHub as a
+    remote to the clone of the restic repository.
  3. Then commit your changes as fine grained as possible, as smaller patches,
     that handle one and only one issue are easier to discuss and merge.
  4. Push the new branch with your changes to your fork of the repository.

Makefile

@@ -1,18 +1,4 @@
-.PHONY: all clean env test bench gox test-integration
-
-TMPGOPATH=$(PWD)/.gopath
-VENDORPATH=$(PWD)/Godeps/_workspace
-BASE=github.com/restic/restic
-BASEPATH=$(TMPGOPATH)/src/$(BASE)
-
-GOPATH=$(TMPGOPATH):$(VENDORPATH)
-
-GOTESTFLAGS ?= -v
-GOX_OS ?= linux darwin openbsd freebsd
-SFTP_PATH ?= /usr/lib/ssh/sftp-server
-
-CMDS=$(patsubst cmd/%,%,$(wildcard cmd/*))
-CMDS_DEBUG=$(patsubst %,%.debug,$(CMDS))
+.PHONY: all clean test
 
 SOURCE=$(wildcard *.go) $(wildcard */*.go) $(wildcard */*/*.go)
 
@@ -20,43 +6,17 @@ export GOPATH GOX_OS
 
 all: restic
 
-.gopath:
-	mkdir -p .gopath/src/github.com/restic
-	ln -snf ../../../.. .gopath/src/github.com/restic/restic
+restic: $(SOURCE)
+	go run build.go
 
-%: cmd/% .gopath $(SOURCE)
-	cd $(BASEPATH) && \
-		go build -a -tags release -ldflags "-s" -o $@ ./$<
-
-%.debug: cmd/% .gopath $(SOURCE)
-	cd $(BASEPATH) && \
-		go build -a -tags debug -ldflags "-s" -o $@ ./$<
+restic.debug: $(SOURCE)
+	go run build.go -tags debug
 
 clean:
-	rm -rf .gopath $(CMDS) $(CMDS_DEBUG) *.cov restic_*
-	go clean ./...
+	rm -rf restic restic.debug
 
-test: .gopath
-	cd $(BASEPATH) && \
-		go test $(GOTESTFLAGS) ./...
+test: $(SOURCE)
+	go run run_tests.go /dev/null
 
-bench: .gopath
-	cd $(BASEPATH) && \
-		go test $(GOTESTFLAGS) -bench ./...
-
-gox: .gopath $(SOURCE)
-	cd $(BASEPATH) && \
-		gox -verbose -os "$(GOX_OS)" ./cmd/restic
-
-all.cov: .gopath $(SOURCE)
-	cd $(BASEPATH) && go run run_tests.go all.cov
-
-env:
-	@echo export GOPATH=\"$(GOPATH)\"
-
-goenv:
-	go env
-
-list: .gopath
-	cd $(BASEPATH) && \
-		go list ./...
+all.cov: $(SOURCE)
+	go run run_tests.go all.cov

README.md

@@ -43,11 +43,10 @@ Restic is a program that does backups right. The design goals are:
 Building
 ========
 
-Install Go/Golang (at least version 1.3), then run `make`, afterwards you'll
-find the binary in the current directory:
+Install Go/Golang (at least version 1.3), then run `go run build.go`,
+afterwards you'll find the binary in the current directory:
 
-    $ make
-    [...]
+    $ go run build.go
 
     $ ./restic --help
     Usage:
@@ -73,7 +72,6 @@ find the binary in the current directory:
       snapshots  show snapshots
       version    display version
 
-
 Contribute and Documentation
 ============================
 
@@ -82,6 +80,13 @@ Contributions are welcome! More information can be found in
 restic and the data structures stored on disc is contained in
 [`doc/Design.md`](doc/Design.md).
 
+Development
+===========
+
+For development, please have a look at [`CONTRIBUTING.md`](CONTRIBUTING.md),
+especially the section "Development Environment". If you have any questions,
+please get in touch!
+
 Contact
 =======
 

build.go

@@ -0,0 +1,279 @@
+// +build ignore
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+var (
+	verbose    bool
+	keepGopath bool
+)
+
+const timeFormat = "2006-01-02 15:04:05"
+
+// specialDir returns true if the file begins with a special character ('.' or '_').
+func specialDir(name string) bool {
+	if name == "." {
+		return false
+	}
+
+	base := filepath.Base(name)
+	return base[0] == '_' || base[0] == '.'
+}
+
+// updateGopath builds a valid GOPATH at dst, with all Go files in src/ copied
+// to dst/prefix/, so calling
+//
+//   updateGopath("/tmp/gopath", "/home/u/restic", "github.com/restic/restic")
+//
+// with "/home/u/restic" containing the file "foo.go" yields the following tree
+// at "/tmp/gopath":
+//
+//   /tmp/gopath
+//   └── src
+//       └── github.com
+//           └── restic
+//               └── restic
+//                   └── foo.go
+func updateGopath(dst, src, prefix string) error {
+	return filepath.Walk(src, func(name string, fi os.FileInfo, err error) error {
+		if specialDir(name) {
+			if fi.IsDir() {
+				return filepath.SkipDir
+			}
+
+			return nil
+		}
+
+		if fi.IsDir() {
+			return nil
+		}
+
+		ext := path.Ext(name)
+		if ext != ".go" && ext != ".s" {
+			return nil
+		}
+
+		intermediatePath, err := filepath.Rel(src, name)
+		if err != nil {
+			return err
+		}
+
+		fileSrc := filepath.Join(src, intermediatePath)
+		fileDst := filepath.Join(dst, "src", prefix, intermediatePath)
+
+		return copyFile(fileDst, fileSrc)
+	})
+}
+
+// copyFile creates dst from src, preserving file attributes and timestamps.
+func copyFile(dst, src string) error {
+	fi, err := os.Stat(src)
+	if err != nil {
+		return err
+	}
+
+	fsrc, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+
+	if err = os.MkdirAll(filepath.Dir(dst), 0755); err != nil {
+		fmt.Printf("MkdirAll(%v)\n", filepath.Dir(dst))
+		return err
+	}
+
+	fdst, err := os.Create(dst)
+	if err != nil {
+		return err
+	}
+
+	if _, err = io.Copy(fdst, fsrc); err != nil {
+		return err
+	}
+
+	if err == nil {
+		err = fsrc.Close()
+	}
+
+	if err == nil {
+		err = fdst.Close()
+	}
+
+	if err == nil {
+		err = os.Chmod(dst, fi.Mode())
+	}
+
+	if err == nil {
+		err = os.Chtimes(dst, fi.ModTime(), fi.ModTime())
+	}
+
+	return nil
+}
+
+// die prints the message with fmt.Fprintf() to stderr and exits with an error
+// code.
+func die(message string, args ...interface{}) {
+	fmt.Fprintf(os.Stderr, message, args...)
+	os.Exit(1)
+}
+
+func showUsage(output io.Writer) {
+	fmt.Fprintf(output, "USAGE: go run build.go OPTIONS\n")
+	fmt.Fprintf(output, "\n")
+	fmt.Fprintf(output, "OPTIONS:\n")
+	fmt.Fprintf(output, "  -v     --verbose   output more messages\n")
+}
+
+func verbosePrintf(message string, args ...interface{}) {
+	if !verbose {
+		return
+	}
+
+	fmt.Printf(message, args...)
+}
+
+// cleanEnv returns a clean environment with GOPATH and GOBIN removed (if
+// present).
+func cleanEnv() (env []string) {
+	for _, v := range os.Environ() {
+		if strings.HasPrefix(v, "GOPATH=") || strings.HasPrefix(v, "GOBIN=") {
+			continue
+		}
+
+		env = append(env, v)
+	}
+
+	return env
+}
+
+// build runs "go build args..." with GOPATH set to gopath.
+func build(gopath string, args ...string) error {
+	args = append([]string{"build"}, args...)
+	cmd := exec.Command("go", args...)
+	cmd.Env = append(cleanEnv(), "GOPATH="+gopath)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	verbosePrintf("go %s\n", args)
+
+	return cmd.Run()
+}
+
+// getVersion returns a version string, either from the file VERSION in the
+// current directory or from git.
+func getVersion() string {
+	v, err := ioutil.ReadFile("VERSION")
+	version := strings.TrimSpace(string(v))
+	if err == nil {
+		verbosePrintf("version from file 'VERSION' is %s\n", version)
+		return version
+	}
+
+	return gitVersion()
+}
+
+// gitVersion returns a version string that identifies the currently checked
+// out git commit.
+func gitVersion() string {
+	cmd := exec.Command("git", "describe",
+		"--long", "--tags", "--dirty", "--always")
+	out, err := cmd.Output()
+	if err != nil {
+		die("git describe returned error: %v\n", err)
+	}
+
+	version := strings.TrimSpace(string(out))
+	verbosePrintf("git version is %s\n", version)
+	return version
+}
+
+func main() {
+	buildTags := []string{}
+
+	skipNext := false
+	params := os.Args[1:]
+	for i, arg := range params {
+		if skipNext {
+			skipNext = false
+			continue
+		}
+
+		switch arg {
+		case "-v", "--verbose":
+			verbose = true
+		case "-k", "--keep-gopath":
+			keepGopath = true
+		case "-t", "-tags", "--tags":
+			skipNext = true
+			buildTags = strings.Split(params[i+1], " ")
+		case "-h":
+			showUsage(os.Stdout)
+		default:
+			fmt.Fprintf(os.Stderr, "Error: unknown option %q\n\n", arg)
+			showUsage(os.Stderr)
+			os.Exit(1)
+		}
+	}
+
+	if len(buildTags) == 0 {
+		verbosePrintf("adding build-tag release\n")
+		buildTags = []string{"release"}
+	}
+
+	for i := range buildTags {
+		buildTags[i] = strings.TrimSpace(buildTags[i])
+	}
+
+	verbosePrintf("build tags: %s\n", buildTags)
+
+	root, err := os.Getwd()
+	if err != nil {
+		die("Getwd(): %v\n", err)
+	}
+
+	gopath, err := ioutil.TempDir("", "restic-build-")
+	if err != nil {
+		die("TempDir(): %v\n", err)
+	}
+
+	verbosePrintf("create GOPATH at %v\n", gopath)
+	if err = updateGopath(gopath, root, "github.com/restic/restic"); err != nil {
+		die("copying files from %v to %v failed: %v\n", root, gopath, err)
+	}
+
+	vendor := filepath.Join(root, "Godeps", "_workspace", "src")
+	if err = updateGopath(gopath, vendor, ""); err != nil {
+		die("copying files from %v to %v failed: %v\n", root, gopath, err)
+	}
+
+	version := getVersion()
+	compileTime := time.Now().Format(timeFormat)
+	args := []string{
+		"-tags", strings.Join(buildTags, " "),
+		"-ldflags", fmt.Sprintf(`-s -X main.version %q -X main.compiledAt %q`, version, compileTime),
+		"-o", "restic", "github.com/restic/restic/cmd/restic",
+	}
+	err = build(gopath, args...)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "build failed: %v\n", err)
+	}
+
+	if !keepGopath {
+		verbosePrintf("remove %v\n", gopath)
+		if err = os.RemoveAll(gopath); err != nil {
+			die("remove GOPATH at %s failed: %v\n", err)
+		}
+	} else {
+		fmt.Printf("leaving temporary GOPATH at %v\n", gopath)
+	}
+}

cmd/restic/Makefile

@@ -1,18 +0,0 @@
-.PHONY: all clean debug
-
-# include config file if it exists
--include $(CURDIR)/config.mk
-
-all: restic
-
-debug: restic.debug
-
-restic: $(wildcard *.go) $(wildcard ../../*.go) $(wildcard ../../*/*.go)
-	go build -a
-
-restic.debug: $(wildcard *.go) $(wildcard ../../*.go) $(wildcard ../../*/*.go)
-	go build -a -tags debug -o restic.debug
-
-clean:
-	go clean
-	rm -f restic restic.debug

cmd/restic/cmd_version.go

@@ -18,7 +18,8 @@ func init() {
 }
 
 func (cmd CmdVersion) Execute(args []string) error {
-	fmt.Printf("restic %s on %v\n", version, runtime.Version())
+	fmt.Printf("restic %s\ncompiled at %s with %v\n",
+		version, compiledAt, runtime.Version())
 
 	return nil
 }

cmd/restic/global.go

@@ -17,6 +17,7 @@ import (
 )
 
 var version = "compiled manually"
+var compiledAt = "unknown time"
 
 type GlobalOptions struct {
 	Repo     string `short:"r" long:"repo"                      description:"Repository directory to backup to/restore from"`

cmd/restic/version.sh

@@ -1,13 +0,0 @@
-#!/bin/sh
-
-VERSION=$(git log --max-count=1 --pretty='%ad-%h' --date=short HEAD 2>/dev/null)
-
-if [ -n "$VERSION" ]; then
-    if ! sh -c "git diff -s --exit-code && git diff --cached -s --exit-code"; then
-        VERSION+="+"
-    fi
-else
-    VERSION="unknown version"
-fi
-
-echo $VERSION