diff --git a/Gopkg.lock b/Gopkg.lock index 54ee83b62..e7cad4964 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -94,8 +94,8 @@ [[projects]] name = "github.com/kurin/blazer" packages = ["b2","base","internal/b2assets","internal/b2types","internal/blog","x/window"] - revision = "7f1134c7489e86be5c924137996d4e421815f48a" - version = "v0.5.0" + revision = "caf65aa76491dc533bac68ad3243ce72fa4e0a0a" + version = "v0.5.1" [[projects]] name = "github.com/marstr/guid" diff --git a/doc/030_preparing_a_new_repo.rst b/doc/030_preparing_a_new_repo.rst index 5c3eea321..be189c235 100644 --- a/doc/030_preparing_a_new_repo.rst +++ b/doc/030_preparing_a_new_repo.rst @@ -293,16 +293,21 @@ Backblaze B2 ************ Restic can backup data to any Backblaze B2 bucket. You need to first setup the -following environment variables with the credentials you obtained when signed -into your B2 account: +following environment variables with the credentials you can find in the +dashboard in on the "Buckets" page when signed into your B2 account: .. code-block:: console $ export B2_ACCOUNT_ID= $ export B2_ACCOUNT_KEY= -You can then easily initialize a repository stored at Backblaze B2. If the -bucket does not exist yet, it will be created: +You can either specify the so-called "Master Application Key" here (which can +access any bucket at any path) or a dedicated "Application Key" created just +for restic (which may be restricted to a specific bucket and/or path). + +You can then initialize a repository stored at Backblaze B2. If the +bucket does not exist yet and the credentials you passed to restic have the +privilege to create buckets, it will be created automatically: .. code-block:: console diff --git a/vendor/github.com/kurin/blazer/b2/backend.go b/vendor/github.com/kurin/blazer/b2/backend.go index bd91fef74..fcaadec31 100644 --- a/vendor/github.com/kurin/blazer/b2/backend.go +++ b/vendor/github.com/kurin/blazer/b2/backend.go @@ -154,6 +154,7 @@ type beKeyInterface interface { name() string expires() time.Time secret() string + id() string } type beKey struct { @@ -711,6 +712,7 @@ func (b *beKey) caps() []string { return b.k.caps() } func (b *beKey) name() string { return b.k.name() } func (b *beKey) expires() time.Time { return b.k.expires() } func (b *beKey) secret() string { return b.k.secret() } +func (b *beKey) id() string { return b.k.id() } func jitter(d time.Duration) time.Duration { f := float64(d) diff --git a/vendor/github.com/kurin/blazer/b2/baseline.go b/vendor/github.com/kurin/blazer/b2/baseline.go index 5ca43e6fa..1de92da13 100644 --- a/vendor/github.com/kurin/blazer/b2/baseline.go +++ b/vendor/github.com/kurin/blazer/b2/baseline.go @@ -105,6 +105,7 @@ type b2KeyInterface interface { name() string expires() time.Time secret() string + id() string } type b2Root struct { @@ -508,3 +509,4 @@ func (b *b2Key) caps() []string { return b.b.Capabilities } func (b *b2Key) name() string { return b.b.Name } func (b *b2Key) expires() time.Time { return b.b.Expires } func (b *b2Key) secret() string { return b.b.Secret } +func (b *b2Key) id() string { return b.b.ID } diff --git a/vendor/github.com/kurin/blazer/b2/integration_test.go b/vendor/github.com/kurin/blazer/b2/integration_test.go index 401becac5..084b71d36 100644 --- a/vendor/github.com/kurin/blazer/b2/integration_test.go +++ b/vendor/github.com/kurin/blazer/b2/integration_test.go @@ -1014,6 +1014,68 @@ func TestVerifyReader(t *testing.T) { } } +func TestListBucketsWithKey(t *testing.T) { + ctx := context.Background() + bucket, done := startLiveTest(ctx, t) + defer done() + + key, err := bucket.CreateKey(ctx, "testKey", Capabilities("listBuckets")) + if err != nil { + t.Fatal(err) + } + + client, err := NewClient(ctx, key.ID(), key.Secret()) + if err != nil { + t.Fatal(err) + } + if _, err := client.Bucket(ctx, bucket.Name()); err != nil { + t.Fatal(err) + } +} + +func TestListBucketContentsWithKey(t *testing.T) { + ctx := context.Background() + bucket, done := startLiveTest(ctx, t) + defer done() + + for _, path := range []string{"foo/bar", "foo/baz", "foo", "bar", "baz"} { + if _, _, err := writeFile(ctx, bucket, path, 1, 1e8); err != nil { + t.Fatal(err) + } + } + + key, err := bucket.CreateKey(ctx, "testKey", Capabilities("listBuckets", "listFiles"), Prefix("foo/")) + if err != nil { + t.Fatal(err) + } + client, err := NewClient(ctx, key.ID(), key.Secret()) + if err != nil { + t.Fatal(err) + } + obucket, err := client.Bucket(ctx, bucket.Name()) + if err != nil { + t.Fatal(err) + } + iter := obucket.List(ctx) + var got []string + for iter.Next() { + got = append(got, iter.Object().Name()) + } + if iter.Err() != nil { + t.Fatal(iter.Err()) + } + want := []string{"foo/bar", "foo/baz"} + if !reflect.DeepEqual(got, want) { + t.Errorf("error listing objects with restricted key: got %v, want %v", got, want) + } + iter2 := obucket.List(ctx, ListHidden()) + for iter2.Next() { + } + if iter2.Err() != nil { + t.Error(iter2.Err()) + } +} + func TestCreateDeleteKey(t *testing.T) { ctx := context.Background() bucket, done := startLiveTest(ctx, t) @@ -1049,9 +1111,7 @@ func TestCreateDeleteKey(t *testing.T) { for _, e := range table { var opts []KeyOption - for _, cap := range e.cap { - opts = append(opts, Capability(cap)) - } + opts = append(opts, Capabilities(e.cap...)) if e.d != 0 { opts = append(opts, Lifetime(e.d)) } diff --git a/vendor/github.com/kurin/blazer/b2/key.go b/vendor/github.com/kurin/blazer/b2/key.go index 950780237..bfb544efa 100644 --- a/vendor/github.com/kurin/blazer/b2/key.go +++ b/vendor/github.com/kurin/blazer/b2/key.go @@ -47,6 +47,10 @@ func (k *Key) Delete(ctx context.Context) error { return k.k.del(ctx) } // operations. func (k *Key) Secret() string { return k.k.secret() } +// ID returns the application key ID. This, plus the secret, is necessary to +// authenticate to B2. +func (k *Key) ID() string { return k.k.id() } + type keyOptions struct { caps []string prefix string @@ -69,10 +73,10 @@ func Deadline(t time.Time) KeyOption { return Lifetime(d) } -// Capability requests a key with the given capability. -func Capability(cap string) KeyOption { +// Capabilities requests a key with the given capability. +func Capabilities(caps ...string) KeyOption { return func(k *keyOptions) { - k.caps = append(k.caps, cap) + k.caps = append(k.caps, caps...) } } diff --git a/vendor/github.com/kurin/blazer/base/base.go b/vendor/github.com/kurin/blazer/base/base.go index db8867ea2..4893c0299 100644 --- a/vendor/github.com/kurin/blazer/base/base.go +++ b/vendor/github.com/kurin/blazer/base/base.go @@ -42,7 +42,7 @@ import ( const ( APIBase = "https://api.backblazeb2.com" - DefaultUserAgent = "blazer/0.5.0" + DefaultUserAgent = "blazer/0.5.1" ) type b2err struct { @@ -268,6 +268,8 @@ type B2 struct { downloadURI string minPartSize int opts *b2Options + bucket string // restricted to this bucket if present + pfx string // restricted to objects with this prefix if present } // Update replaces the B2 object with a new one, in-place. @@ -428,6 +430,8 @@ func AuthorizeAccount(ctx context.Context, account, key string, opts ...AuthOpti apiURI: b2resp.URI, downloadURI: b2resp.DownloadURI, minPartSize: b2resp.PartSize, + bucket: b2resp.Allowed.Bucket, + pfx: b2resp.Allowed.Prefix, opts: b2opts, }, nil } @@ -614,6 +618,7 @@ func (b *Bucket) BaseURL() string { func (b *B2) ListBuckets(ctx context.Context) ([]*Bucket, error) { b2req := &b2types.ListBucketsRequest{ AccountID: b.accountID, + Bucket: b.bucket, } b2resp := &b2types.ListBucketsResponse{} headers := map[string]string{ @@ -967,6 +972,9 @@ func (b *Bucket) ListUnfinishedLargeFiles(ctx context.Context, count int, contin // ListFileNames wraps b2_list_file_names. func (b *Bucket) ListFileNames(ctx context.Context, count int, continuation, prefix, delimiter string) ([]*File, string, error) { + if prefix == "" { + prefix = b.b2.pfx + } b2req := &b2types.ListFileNamesRequest{ Count: count, Continuation: continuation, @@ -1007,6 +1015,9 @@ func (b *Bucket) ListFileNames(ctx context.Context, count int, continuation, pre // ListFileVersions wraps b2_list_file_versions. func (b *Bucket) ListFileVersions(ctx context.Context, count int, startName, startID, prefix, delimiter string) ([]*File, string, string, error) { + if prefix == "" { + prefix = b.b2.pfx + } b2req := &b2types.ListFileVersionsRequest{ BucketID: b.ID, Count: count, diff --git a/vendor/github.com/kurin/blazer/bin/b2keys/b2keys.go b/vendor/github.com/kurin/blazer/bin/b2keys/b2keys.go index 2a29bb8a6..a08203617 100644 --- a/vendor/github.com/kurin/blazer/bin/b2keys/b2keys.go +++ b/vendor/github.com/kurin/blazer/bin/b2keys/b2keys.go @@ -65,9 +65,7 @@ func (c *create) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) if *c.pfx != "" { opts = append(opts, b2.Prefix(*c.pfx)) } - for _, c := range caps { - opts = append(opts, b2.Capability(c)) - } + opts = append(opts, b2.Capabilities(caps...)) client, err := b2.NewClient(ctx, id, key, b2.UserAgent("b2keys")) if err != nil { @@ -86,10 +84,12 @@ func (c *create) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) cr = bucket } - if _, err := cr.CreateKey(ctx, name, opts...); err != nil { + b2key, err := cr.CreateKey(ctx, name, opts...) + if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) return subcommands.ExitFailure } + fmt.Printf("key=%s, secret=%s\n", b2key.ID(), b2key.Secret()) return subcommands.ExitSuccess } diff --git a/vendor/github.com/kurin/blazer/internal/b2types/b2types.go b/vendor/github.com/kurin/blazer/internal/b2types/b2types.go index 3523e4148..d70061f77 100644 --- a/vendor/github.com/kurin/blazer/internal/b2types/b2types.go +++ b/vendor/github.com/kurin/blazer/internal/b2types/b2types.go @@ -29,14 +29,20 @@ type ErrorMessage struct { } type AuthorizeAccountResponse struct { - AccountID string `json:"accountId"` - AuthToken string `json:"authorizationToken"` - URI string `json:"apiUrl"` - DownloadURI string `json:"downloadUrl"` - MinPartSize int `json:"minimumPartSize"` - PartSize int `json:"recommendedPartSize"` - AbsMinPartSize int `json:"absoluteMinimumPartSize"` - Capabilities []string `json:"capabilities"` + AccountID string `json:"accountId"` + AuthToken string `json:"authorizationToken"` + URI string `json:"apiUrl"` + DownloadURI string `json:"downloadUrl"` + MinPartSize int `json:"minimumPartSize"` + PartSize int `json:"recommendedPartSize"` + AbsMinPartSize int `json:"absoluteMinimumPartSize"` + Allowed Allowance `json:"allowed"` +} + +type Allowance struct { + Capabilities []string `json:"capabilities"` + Bucket string `json:"bucketId"` + Prefix string `json:"namePrefix"` } type LifecycleRule struct { @@ -69,6 +75,7 @@ type DeleteBucketRequest struct { type ListBucketsRequest struct { AccountID string `json:"accountId"` + Bucket string `json:"bucketId,omitempty"` } type ListBucketsResponse struct { diff --git a/vendor/github.com/kurin/blazer/internal/pyre/api.go b/vendor/github.com/kurin/blazer/internal/pyre/api.go index ba17c0c38..df0a689fc 100644 --- a/vendor/github.com/kurin/blazer/internal/pyre/api.go +++ b/vendor/github.com/kurin/blazer/internal/pyre/api.go @@ -255,9 +255,9 @@ func (s *Server) ListFileVersions(ctx context.Context, req *pb.ListFileVersionsR return nil, nil } -//type objTuple struct { -// name, version string -//} +type objTuple struct { + name, version string +} type ListManager interface { // NextN returns the next n objects, sorted by lexicographical order by name, @@ -276,6 +276,35 @@ type VersionedObject interface { NextNVersions(begin string, n int) ([]string, error) } +func getDirNames(lm ListManager, bucket, name, prefix, delim string, n int) ([]string, error) { + var sfx string + var out []string + for n > 0 { + vo, err := lm.NextN(bucket, name, prefix, sfx, 1) + if err != nil { + return nil, err + } + if len(vo) == 0 { + return out, nil + } + v := vo[0] + name = v.Name() + suffix := name[len(prefix):] + i := strings.Index(suffix, delim) + if i < 0 { + sfx = "" + out = append(out, name) + name += "\000" + n-- + continue + } + sfx = v.Name()[:len(prefix)+i+1] + out = append(out, sfx) + n-- + } + return out, nil +} + //func getNextObjects(lm ListManager, bucket, name, prefix, delimiter string, n int) ([]VersionedObject, error) { // if delimiter == "" { // return lm.NextN(bucket, name, prefix, "", n) diff --git a/vendor/github.com/kurin/blazer/internal/pyre/api_test.go b/vendor/github.com/kurin/blazer/internal/pyre/api_test.go new file mode 100644 index 000000000..549341529 --- /dev/null +++ b/vendor/github.com/kurin/blazer/internal/pyre/api_test.go @@ -0,0 +1,137 @@ +// Copyright 2018, Google +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pyre + +import ( + "reflect" + "sort" + "strings" + "sync" + "testing" +) + +type testVersionedObject struct { + name string + versions []string +} + +func (t testVersionedObject) Name() string { return t.name } + +func (t testVersionedObject) NextNVersions(b string, n int) ([]string, error) { + var out []string + var seen bool + if b == "" { + seen = true + } + for _, v := range t.versions { + if b == v { + seen = true + } + if !seen { + continue + } + if len(out) >= n { + return out, nil + } + out = append(out, v) + } + return out, nil +} + +type testListManager struct { + objs map[string][]string + m sync.Mutex +} + +func (t *testListManager) NextN(b, fn, pfx, spfx string, n int) ([]VersionedObject, error) { + t.m.Lock() + defer t.m.Unlock() + + var out []VersionedObject + var keys []string + for k := range t.objs { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if k < fn { + continue + } + if !strings.HasPrefix(k, pfx) { + continue + } + if spfx != "" && strings.HasPrefix(k, spfx) { + continue + } + out = append(out, testVersionedObject{name: k, versions: t.objs[k]}) + n-- + if n <= 0 { + return out, nil + } + } + return out, nil +} + +func TestGetDirNames(t *testing.T) { + table := []struct { + lm ListManager + name string + pfx string + delim string + num int + want []string + }{ + { + lm: &testListManager{ + objs: map[string][]string{ + "/usr/local/etc/foo/bar": {"a"}, + "/usr/local/etc/foo/baz": {"a"}, + "/usr/local/etc/foo": {"a"}, + "/usr/local/etc/fool": {"a"}, + }, + }, + num: 2, + pfx: "/usr/local/etc/", + delim: "/", + want: []string{"/usr/local/etc/foo", "/usr/local/etc/foo/"}, + }, + { + lm: &testListManager{ + objs: map[string][]string{ + "/usr/local/etc/foo/bar": {"a"}, + "/usr/local/etc/foo/baz": {"a"}, + "/usr/local/etc/foo": {"a"}, + "/usr/local/etc/fool": {"a"}, + "/usr/local/etc/bar": {"a"}, + }, + }, + num: 4, + pfx: "/usr/local/etc/", + delim: "/", + want: []string{"/usr/local/etc/bar", "/usr/local/etc/foo", "/usr/local/etc/foo/", "/usr/local/etc/fool"}, + }, + } + + for _, e := range table { + got, err := getDirNames(e.lm, "", e.name, e.pfx, e.delim, e.num) + if err != nil { + t.Error(err) + continue + } + if !reflect.DeepEqual(got, e.want) { + t.Errorf("getDirNames(%v, %q, %q, %q, %d): got %v, want %v", e.lm, e.name, e.pfx, e.delim, e.num, got, e.want) + } + } +}