mirror of
https://github.com/go-gitea/gitea.git
synced 2024-08-24 13:17:28 +02:00
This PR reworked the Find pointer files feature in Settings -> LFS page. When a LFS object is missing from database but exists in LFS content store, admin can associate it to the repository by clicking the Associate button. This PR is not perfect (because the LFS module itself should be improved too), it's just a nice-to-have feature to help users recover their LFS repositories (eg: database was lost / table was truncated)
248 lines
7.1 KiB
Go
248 lines
7.1 KiB
Go
// Copyright 2020 The Gitea Authors. All rights reserved.
|
|
// Use of this source code is governed by a MIT-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
package models
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
|
|
"code.gitea.io/gitea/models/db"
|
|
repo_model "code.gitea.io/gitea/models/repo"
|
|
user_model "code.gitea.io/gitea/models/user"
|
|
"code.gitea.io/gitea/modules/lfs"
|
|
"code.gitea.io/gitea/modules/log"
|
|
"code.gitea.io/gitea/modules/timeutil"
|
|
|
|
"xorm.io/builder"
|
|
)
|
|
|
|
// LFSMetaObject stores metadata for LFS tracked files.
|
|
type LFSMetaObject struct {
|
|
ID int64 `xorm:"pk autoincr"`
|
|
lfs.Pointer `xorm:"extends"`
|
|
RepositoryID int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
|
|
Existing bool `xorm:"-"`
|
|
CreatedUnix timeutil.TimeStamp `xorm:"created"`
|
|
}
|
|
|
|
func init() {
|
|
db.RegisterModel(new(LFSMetaObject))
|
|
}
|
|
|
|
// LFSTokenResponse defines the JSON structure in which the JWT token is stored.
|
|
// This structure is fetched via SSH and passed by the Git LFS client to the server
|
|
// endpoint for authorization.
|
|
type LFSTokenResponse struct {
|
|
Header map[string]string `json:"header"`
|
|
Href string `json:"href"`
|
|
}
|
|
|
|
// ErrLFSObjectNotExist is returned from lfs models functions in order
|
|
// to differentiate between database and missing object errors.
|
|
var ErrLFSObjectNotExist = errors.New("LFS Meta object does not exist")
|
|
|
|
// NewLFSMetaObject stores a given populated LFSMetaObject structure in the database
|
|
// if it is not already present.
|
|
func NewLFSMetaObject(m *LFSMetaObject) (*LFSMetaObject, error) {
|
|
var err error
|
|
|
|
ctx, committer, err := db.TxContext()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer committer.Close()
|
|
|
|
has, err := db.GetByBean(ctx, m)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if has {
|
|
m.Existing = true
|
|
return m, committer.Commit()
|
|
}
|
|
|
|
if err = db.Insert(ctx, m); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return m, committer.Commit()
|
|
}
|
|
|
|
// GetLFSMetaObjectByOid selects a LFSMetaObject entry from database by its OID.
|
|
// It may return ErrLFSObjectNotExist or a database error. If the error is nil,
|
|
// the returned pointer is a valid LFSMetaObject.
|
|
func GetLFSMetaObjectByOid(repoID int64, oid string) (*LFSMetaObject, error) {
|
|
if len(oid) == 0 {
|
|
return nil, ErrLFSObjectNotExist
|
|
}
|
|
|
|
m := &LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}, RepositoryID: repoID}
|
|
has, err := db.GetEngine(db.DefaultContext).Get(m)
|
|
if err != nil {
|
|
return nil, err
|
|
} else if !has {
|
|
return nil, ErrLFSObjectNotExist
|
|
}
|
|
return m, nil
|
|
}
|
|
|
|
// RemoveLFSMetaObjectByOid removes a LFSMetaObject entry from database by its OID.
|
|
// It may return ErrLFSObjectNotExist or a database error.
|
|
func RemoveLFSMetaObjectByOid(repoID int64, oid string) (int64, error) {
|
|
if len(oid) == 0 {
|
|
return 0, ErrLFSObjectNotExist
|
|
}
|
|
|
|
ctx, committer, err := db.TxContext()
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer committer.Close()
|
|
|
|
m := &LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}, RepositoryID: repoID}
|
|
if _, err := db.DeleteByBean(ctx, m); err != nil {
|
|
return -1, err
|
|
}
|
|
|
|
count, err := db.CountByBean(ctx, &LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}})
|
|
if err != nil {
|
|
return count, err
|
|
}
|
|
|
|
return count, committer.Commit()
|
|
}
|
|
|
|
// GetLFSMetaObjects returns all LFSMetaObjects associated with a repository
|
|
func GetLFSMetaObjects(repoID int64, page, pageSize int) ([]*LFSMetaObject, error) {
|
|
sess := db.GetEngine(db.DefaultContext)
|
|
|
|
if page >= 0 && pageSize > 0 {
|
|
start := 0
|
|
if page > 0 {
|
|
start = (page - 1) * pageSize
|
|
}
|
|
sess.Limit(pageSize, start)
|
|
}
|
|
lfsObjects := make([]*LFSMetaObject, 0, pageSize)
|
|
return lfsObjects, sess.Find(&lfsObjects, &LFSMetaObject{RepositoryID: repoID})
|
|
}
|
|
|
|
// CountLFSMetaObjects returns a count of all LFSMetaObjects associated with a repository
|
|
func CountLFSMetaObjects(repoID int64) (int64, error) {
|
|
return db.GetEngine(db.DefaultContext).Count(&LFSMetaObject{RepositoryID: repoID})
|
|
}
|
|
|
|
// LFSObjectAccessible checks if a provided Oid is accessible to the user
|
|
func LFSObjectAccessible(user *user_model.User, oid string) (bool, error) {
|
|
if user.IsAdmin {
|
|
count, err := db.GetEngine(db.DefaultContext).Count(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}})
|
|
return count > 0, err
|
|
}
|
|
cond := accessibleRepositoryCondition(user)
|
|
count, err := db.GetEngine(db.DefaultContext).Where(cond).Join("INNER", "repository", "`lfs_meta_object`.repository_id = `repository`.id").Count(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}})
|
|
return count > 0, err
|
|
}
|
|
|
|
// LFSObjectIsAssociated checks if a provided Oid is associated
|
|
func LFSObjectIsAssociated(oid string) (bool, error) {
|
|
return db.GetEngine(db.DefaultContext).Exist(&LFSMetaObject{Pointer: lfs.Pointer{Oid: oid}})
|
|
}
|
|
|
|
// LFSAutoAssociate auto associates accessible LFSMetaObjects
|
|
func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int64) error {
|
|
ctx, committer, err := db.TxContext()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer committer.Close()
|
|
|
|
sess := db.GetEngine(ctx)
|
|
|
|
oids := make([]interface{}, len(metas))
|
|
oidMap := make(map[string]*LFSMetaObject, len(metas))
|
|
for i, meta := range metas {
|
|
oids[i] = meta.Oid
|
|
oidMap[meta.Oid] = meta
|
|
}
|
|
|
|
if !user.IsAdmin {
|
|
newMetas := make([]*LFSMetaObject, 0, len(metas))
|
|
cond := builder.In(
|
|
"`lfs_meta_object`.repository_id",
|
|
builder.Select("`repository`.id").From("repository").Where(accessibleRepositoryCondition(user)),
|
|
)
|
|
err = sess.Cols("oid").Where(cond).In("oid", oids...).GroupBy("oid").Find(&newMetas)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(newMetas) != len(oidMap) {
|
|
return fmt.Errorf("unable collect all LFS objects from database, expected %d, actually %d", len(oidMap), len(newMetas))
|
|
}
|
|
for i := range newMetas {
|
|
newMetas[i].Size = oidMap[newMetas[i].Oid].Size
|
|
newMetas[i].RepositoryID = repoID
|
|
}
|
|
if err = db.Insert(ctx, newMetas); err != nil {
|
|
return err
|
|
}
|
|
} else {
|
|
// admin can associate any LFS object to any repository, and we do not care about errors (eg: duplicated unique key),
|
|
// even if error occurs, it won't hurt users and won't make things worse
|
|
for i := range metas {
|
|
_, err = sess.Insert(&LFSMetaObject{
|
|
Pointer: lfs.Pointer{Oid: metas[i].Oid, Size: metas[i].Size},
|
|
RepositoryID: repoID,
|
|
})
|
|
if err != nil {
|
|
log.Warn("failed to insert LFS meta object into database, err=%v", err)
|
|
}
|
|
}
|
|
}
|
|
return committer.Commit()
|
|
}
|
|
|
|
// IterateLFS iterates lfs object
|
|
func IterateLFS(f func(mo *LFSMetaObject) error) error {
|
|
var start int
|
|
const batchSize = 100
|
|
var e = db.GetEngine(db.DefaultContext)
|
|
for {
|
|
mos := make([]*LFSMetaObject, 0, batchSize)
|
|
if err := e.Limit(batchSize, start).Find(&mos); err != nil {
|
|
return err
|
|
}
|
|
if len(mos) == 0 {
|
|
return nil
|
|
}
|
|
start += len(mos)
|
|
|
|
for _, mo := range mos {
|
|
if err := f(mo); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// CopyLFS copies LFS data from one repo to another
|
|
func CopyLFS(ctx context.Context, newRepo, oldRepo *repo_model.Repository) error {
|
|
var lfsObjects []*LFSMetaObject
|
|
if err := db.GetEngine(ctx).Where("repository_id=?", oldRepo.ID).Find(&lfsObjects); err != nil {
|
|
return err
|
|
}
|
|
|
|
for _, v := range lfsObjects {
|
|
v.ID = 0
|
|
v.RepositoryID = newRepo.ID
|
|
if _, err := db.GetEngine(ctx).Insert(v); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|