mirror of
https://github.com/restic/restic.git
synced 2024-08-15 00:43:22 +02:00
232 lines
8.4 KiB
Go
232 lines
8.4 KiB
Go
|
// Copyright 2016 Google Inc. All Rights Reserved.
|
||
|
//
|
||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||
|
// you may not use this file except in compliance with the License.
|
||
|
// You may obtain a copy of the License at
|
||
|
//
|
||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||
|
//
|
||
|
// Unless required by applicable law or agreed to in writing, software
|
||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
// See the License for the specific language governing permissions and
|
||
|
// limitations under the License.
|
||
|
|
||
|
// Package iterator provides support for standard Google API iterators.
|
||
|
// See https://github.com/GoogleCloudPlatform/gcloud-golang/wiki/Iterator-Guidelines.
|
||
|
package iterator
|
||
|
|
||
|
import (
|
||
|
"errors"
|
||
|
"fmt"
|
||
|
"reflect"
|
||
|
)
|
||
|
|
||
|
// Done is returned by an iterator's Next method when the iteration is
|
||
|
// complete; when there are no more items to return.
|
||
|
var Done = errors.New("no more items in iterator")
|
||
|
|
||
|
// We don't support mixed calls to Next and NextPage because they play
|
||
|
// with the paging state in incompatible ways.
|
||
|
var errMixed = errors.New("iterator: Next and NextPage called on same iterator")
|
||
|
|
||
|
// PageInfo contains information about an iterator's paging state.
|
||
|
type PageInfo struct {
|
||
|
// Token is the token used to retrieve the next page of items from the
|
||
|
// API. You may set Token immediately after creating an iterator to
|
||
|
// begin iteration at a particular point. If Token is the empty string,
|
||
|
// the iterator will begin with the first eligible item.
|
||
|
//
|
||
|
// The result of setting Token after the first call to Next is undefined.
|
||
|
//
|
||
|
// After the underlying API method is called to retrieve a page of items,
|
||
|
// Token is set to the next-page token in the response.
|
||
|
Token string
|
||
|
|
||
|
// MaxSize is the maximum number of items returned by a call to the API.
|
||
|
// Set MaxSize as a hint to optimize the buffering behavior of the iterator.
|
||
|
// If zero, the page size is determined by the underlying service.
|
||
|
//
|
||
|
// Use Pager to retrieve a page of a specific, exact size.
|
||
|
MaxSize int
|
||
|
|
||
|
// The error state of the iterator. Manipulated by PageInfo.next and Pager.
|
||
|
// This is a latch: it starts as nil, and once set should never change.
|
||
|
err error
|
||
|
|
||
|
// If true, no more calls to fetch should be made. Set to true when fetch
|
||
|
// returns an empty page token. The iterator is Done when this is true AND
|
||
|
// the buffer is empty.
|
||
|
atEnd bool
|
||
|
|
||
|
// Function that fetches a page from the underlying service. It should pass
|
||
|
// the pageSize and pageToken arguments to the service, fill the buffer
|
||
|
// with the results from the call, and return the next-page token returned
|
||
|
// by the service. The function must not remove any existing items from the
|
||
|
// buffer. If the underlying RPC takes an int32 page size, pageSize should
|
||
|
// be silently truncated.
|
||
|
fetch func(pageSize int, pageToken string) (nextPageToken string, err error)
|
||
|
|
||
|
// Function that clears the iterator's buffer, returning any currently buffered items.
|
||
|
bufLen func() int
|
||
|
|
||
|
// Function that returns the buffer, after setting the buffer variable to nil.
|
||
|
takeBuf func() interface{}
|
||
|
|
||
|
// Set to true on first call to PageInfo.next or Pager.NextPage. Used to check
|
||
|
// for calls to both Next and NextPage with the same iterator.
|
||
|
nextCalled, nextPageCalled bool
|
||
|
}
|
||
|
|
||
|
// NewPageInfo exposes internals for iterator implementations.
|
||
|
// It is not a stable interface.
|
||
|
var NewPageInfo = newPageInfo
|
||
|
|
||
|
// If an iterator can support paging, its iterator-creating method should call
|
||
|
// this (via the NewPageInfo variable above).
|
||
|
//
|
||
|
// The fetch, bufLen and takeBuf arguments provide access to the
|
||
|
// iterator's internal slice of buffered items. They behave as described in
|
||
|
// PageInfo, above.
|
||
|
//
|
||
|
// The return value is the PageInfo.next method bound to the returned PageInfo value.
|
||
|
// (Returning it avoids exporting PageInfo.next.)
|
||
|
func newPageInfo(fetch func(int, string) (string, error), bufLen func() int, takeBuf func() interface{}) (*PageInfo, func() error) {
|
||
|
pi := &PageInfo{
|
||
|
fetch: fetch,
|
||
|
bufLen: bufLen,
|
||
|
takeBuf: takeBuf,
|
||
|
}
|
||
|
return pi, pi.next
|
||
|
}
|
||
|
|
||
|
// Remaining returns the number of items available before the iterator makes another API call.
|
||
|
func (pi *PageInfo) Remaining() int { return pi.bufLen() }
|
||
|
|
||
|
// next provides support for an iterator's Next function. An iterator's Next
|
||
|
// should return the error returned by next if non-nil; else it can assume
|
||
|
// there is at least one item in its buffer, and it should return that item and
|
||
|
// remove it from the buffer.
|
||
|
func (pi *PageInfo) next() error {
|
||
|
pi.nextCalled = true
|
||
|
if pi.err != nil { // Once we get an error, always return it.
|
||
|
// TODO(jba): fix so users can retry on transient errors? Probably not worth it.
|
||
|
return pi.err
|
||
|
}
|
||
|
if pi.nextPageCalled {
|
||
|
pi.err = errMixed
|
||
|
return pi.err
|
||
|
}
|
||
|
// Loop until we get some items or reach the end.
|
||
|
for pi.bufLen() == 0 && !pi.atEnd {
|
||
|
if err := pi.fill(pi.MaxSize); err != nil {
|
||
|
pi.err = err
|
||
|
return pi.err
|
||
|
}
|
||
|
if pi.Token == "" {
|
||
|
pi.atEnd = true
|
||
|
}
|
||
|
}
|
||
|
// Either the buffer is non-empty or pi.atEnd is true (or both).
|
||
|
if pi.bufLen() == 0 {
|
||
|
// The buffer is empty and pi.atEnd is true, i.e. the service has no
|
||
|
// more items.
|
||
|
pi.err = Done
|
||
|
}
|
||
|
return pi.err
|
||
|
}
|
||
|
|
||
|
// Call the service to fill the buffer, using size and pi.Token. Set pi.Token to the
|
||
|
// next-page token returned by the call.
|
||
|
// If fill returns a non-nil error, the buffer will be empty.
|
||
|
func (pi *PageInfo) fill(size int) error {
|
||
|
tok, err := pi.fetch(size, pi.Token)
|
||
|
if err != nil {
|
||
|
pi.takeBuf() // clear the buffer
|
||
|
return err
|
||
|
}
|
||
|
pi.Token = tok
|
||
|
return nil
|
||
|
}
|
||
|
|
||
|
// Pageable is implemented by iterators that support paging.
|
||
|
type Pageable interface {
|
||
|
// PageInfo returns paging information associated with the iterator.
|
||
|
PageInfo() *PageInfo
|
||
|
}
|
||
|
|
||
|
// Pager supports retrieving iterator items a page at a time.
|
||
|
type Pager struct {
|
||
|
pageInfo *PageInfo
|
||
|
pageSize int
|
||
|
}
|
||
|
|
||
|
// NewPager returns a pager that uses iter. Calls to its NextPage method will
|
||
|
// obtain exactly pageSize items, unless fewer remain. The pageToken argument
|
||
|
// indicates where to start the iteration. Pass the empty string to start at
|
||
|
// the beginning, or pass a token retrieved from a call to Pager.NextPage.
|
||
|
//
|
||
|
// If you use an iterator with a Pager, you must not call Next on the iterator.
|
||
|
func NewPager(iter Pageable, pageSize int, pageToken string) *Pager {
|
||
|
p := &Pager{
|
||
|
pageInfo: iter.PageInfo(),
|
||
|
pageSize: pageSize,
|
||
|
}
|
||
|
p.pageInfo.Token = pageToken
|
||
|
if pageSize <= 0 {
|
||
|
p.pageInfo.err = errors.New("iterator: page size must be positive")
|
||
|
}
|
||
|
return p
|
||
|
}
|
||
|
|
||
|
// NextPage retrieves a sequence of items from the iterator and appends them
|
||
|
// to slicep, which must be a pointer to a slice of the iterator's item type.
|
||
|
// Exactly p.pageSize items will be appended, unless fewer remain.
|
||
|
//
|
||
|
// The first return value is the page token to use for the next page of items.
|
||
|
// If empty, there are no more pages. Aside from checking for the end of the
|
||
|
// iteration, the returned page token is only needed if the iteration is to be
|
||
|
// resumed a later time, in another context (possibly another process).
|
||
|
//
|
||
|
// The second return value is non-nil if an error occurred. It will never be
|
||
|
// the special iterator sentinel value Done. To recognize the end of the
|
||
|
// iteration, compare nextPageToken to the empty string.
|
||
|
//
|
||
|
// It is possible for NextPage to return a single zero-length page along with
|
||
|
// an empty page token when there are no more items in the iteration.
|
||
|
func (p *Pager) NextPage(slicep interface{}) (nextPageToken string, err error) {
|
||
|
p.pageInfo.nextPageCalled = true
|
||
|
if p.pageInfo.err != nil {
|
||
|
return "", p.pageInfo.err
|
||
|
}
|
||
|
if p.pageInfo.nextCalled {
|
||
|
p.pageInfo.err = errMixed
|
||
|
return "", p.pageInfo.err
|
||
|
}
|
||
|
if p.pageInfo.bufLen() > 0 {
|
||
|
return "", errors.New("must call NextPage with an empty buffer")
|
||
|
}
|
||
|
// The buffer must be empty here, so takeBuf is a no-op. We call it just to get
|
||
|
// the buffer's type.
|
||
|
wantSliceType := reflect.PtrTo(reflect.ValueOf(p.pageInfo.takeBuf()).Type())
|
||
|
if slicep == nil {
|
||
|
return "", errors.New("nil passed to Pager.NextPage")
|
||
|
}
|
||
|
vslicep := reflect.ValueOf(slicep)
|
||
|
if vslicep.Type() != wantSliceType {
|
||
|
return "", fmt.Errorf("slicep should be of type %s, got %T", wantSliceType, slicep)
|
||
|
}
|
||
|
for p.pageInfo.bufLen() < p.pageSize {
|
||
|
if err := p.pageInfo.fill(p.pageSize - p.pageInfo.bufLen()); err != nil {
|
||
|
p.pageInfo.err = err
|
||
|
return "", p.pageInfo.err
|
||
|
}
|
||
|
if p.pageInfo.Token == "" {
|
||
|
break
|
||
|
}
|
||
|
}
|
||
|
e := vslicep.Elem()
|
||
|
e.Set(reflect.AppendSlice(e, reflect.ValueOf(p.pageInfo.takeBuf())))
|
||
|
return p.pageInfo.Token, nil
|
||
|
}
|