package repository import ( "encoding/json" "errors" "fmt" "io" "sync" "time" "github.com/restic/restic/backend" "github.com/restic/restic/crypto" "github.com/restic/restic/debug" "github.com/restic/restic/pack" ) // Index holds a lookup table for id -> pack. type Index struct { m sync.Mutex pack map[backend.ID]indexEntry final bool // set to true for all indexes read from the backend ("finalized") id backend.ID // set to the ID of the index when it's finalized supersedes backend.IDs created time.Time } type indexEntry struct { tpe pack.BlobType packID backend.ID offset uint length uint } // NewIndex returns a new index. func NewIndex() *Index { return &Index{ pack: make(map[backend.ID]indexEntry), created: time.Now(), } } func (idx *Index) store(blob PackedBlob) { idx.pack[blob.ID] = indexEntry{ tpe: blob.Type, packID: blob.PackID, offset: blob.Offset, length: blob.Length, } } // Final returns true iff the index is already written to the repository, it is // finalized. func (idx *Index) Final() bool { idx.m.Lock() defer idx.m.Unlock() return idx.final } const ( indexMinBlobs = 20 indexMaxBlobs = 2000 indexMinAge = 2 * time.Minute indexMaxAge = 15 * time.Minute ) // IndexFull returns true iff the index is "full enough" to be saved as a preliminary index. var IndexFull = func(idx *Index) bool { idx.m.Lock() defer idx.m.Unlock() debug.Log("Index.Full", "checking whether index %p is full", idx) packs := len(idx.pack) age := time.Now().Sub(idx.created) if age > indexMaxAge { debug.Log("Index.Full", "index %p is old enough", idx, age) return true } if packs < indexMinBlobs || age < indexMinAge { debug.Log("Index.Full", "index %p only has %d packs or is too young (%v)", idx, packs, age) return false } if packs > indexMaxBlobs { debug.Log("Index.Full", "index %p has %d packs", idx, packs) return true } debug.Log("Index.Full", "index %p is not full", idx) return false } // Store remembers the id and pack in the index. An existing entry will be // silently overwritten. func (idx *Index) Store(blob PackedBlob) { idx.m.Lock() defer idx.m.Unlock() if idx.final { panic("store new item in finalized index") } debug.Log("Index.Store", "%v", blob) idx.store(blob) } // StoreBlobs saves information about the blobs to the index in one atomic transaction. func (idx *Index) StoreBlobs(blobs []PackedBlob) { idx.m.Lock() defer idx.m.Unlock() if idx.final { panic("store new item in finalized index") } debug.Log("Index.StoreBlobs", "stored %d blobs", len(blobs)) for _, blob := range blobs { idx.store(blob) } } // Lookup queries the index for the blob ID and returns a PackedBlob. func (idx *Index) Lookup(id backend.ID) (pb PackedBlob, err error) { idx.m.Lock() defer idx.m.Unlock() if p, ok := idx.pack[id]; ok { debug.Log("Index.Lookup", "id %v found in pack %v at %d, length %d", id.Str(), p.packID.Str(), p.offset, p.length) pb := PackedBlob{ Type: p.tpe, Length: p.length, ID: id, Offset: p.offset, PackID: p.packID, } return pb, nil } debug.Log("Index.Lookup", "id %v not found", id.Str()) return PackedBlob{}, fmt.Errorf("id %v not found in index", id) } // ListPack returns a list of blobs contained in a pack. func (idx *Index) ListPack(id backend.ID) (list []PackedBlob) { idx.m.Lock() defer idx.m.Unlock() for blobID, entry := range idx.pack { if entry.packID == id { list = append(list, PackedBlob{ ID: blobID, Type: entry.tpe, Length: entry.length, Offset: entry.offset, PackID: entry.packID, }) } } return list } // Has returns true iff the id is listed in the index. func (idx *Index) Has(id backend.ID) bool { _, err := idx.Lookup(id) if err == nil { return true } return false } // LookupSize returns the length of the cleartext content behind the // given id func (idx *Index) LookupSize(id backend.ID) (cleartextLength uint, err error) { blob, err := idx.Lookup(id) if err != nil { return 0, err } return blob.PlaintextLength(), nil } // Merge loads all items from other into idx. func (idx *Index) Merge(other *Index) { debug.Log("Index.Merge", "Merge index with %p", other) idx.m.Lock() defer idx.m.Unlock() for k, v := range other.pack { if _, ok := idx.pack[k]; ok { debug.Log("Index.Merge", "index already has key %v, updating", k.Str()) } idx.pack[k] = v } debug.Log("Index.Merge", "done merging index") } // Supersedes returns the list of indexes this index supersedes, if any. func (idx *Index) Supersedes() backend.IDs { return idx.supersedes } // AddToSupersedes adds the ids to the list of indexes superseded by this // index. If the index has already been finalized, an error is returned. func (idx *Index) AddToSupersedes(ids ...backend.ID) error { idx.m.Lock() defer idx.m.Unlock() if idx.final { return errors.New("index already finalized") } idx.supersedes = append(idx.supersedes, ids...) return nil } // PackedBlob is a blob already saved within a pack. type PackedBlob struct { Type pack.BlobType Length uint ID backend.ID Offset uint PackID backend.ID } func (pb PackedBlob) String() string { return fmt.Sprintf("