Move restore repo to internal router and invoke from command to avoid open the same db file or queues files (#15790)

* Move restore repo to internal router and invoke from command to avoid open the same db file or queues files

* Follow @zeripath's review

* set no timeout for resotre repo private request

* make restore repo cancelable
This commit is contained in:
Lunny Xiao 2021-05-10 15:57:45 +08:00 committed by GitHub
parent 1e6fa57acb
commit e5723d6556
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 164 additions and 77 deletions

View File

@ -5,15 +5,12 @@
package cmd package cmd
import ( import (
"context" "errors"
"strings" "net/http"
"code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations" "code.gitea.io/gitea/modules/private"
"code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
pull_service "code.gitea.io/gitea/services/pull"
"github.com/urfave/cli" "github.com/urfave/cli"
) )
@ -50,70 +47,18 @@ wiki, issues, labels, releases, release_assets, milestones, pull_requests, comme
} }
func runRestoreRepository(ctx *cli.Context) error { func runRestoreRepository(ctx *cli.Context) error {
if err := initDB(); err != nil { setting.NewContext()
return err
}
log.Trace("AppPath: %s", setting.AppPath) statusCode, errStr := private.RestoreRepo(
log.Trace("AppWorkPath: %s", setting.AppWorkPath)
log.Trace("Custom path: %s", setting.CustomPath)
log.Trace("Log path: %s", setting.LogRootPath)
setting.InitDBConfig()
if err := storage.Init(); err != nil {
return err
}
if err := pull_service.Init(); err != nil {
return err
}
var opts = base.MigrateOptions{
RepoName: ctx.String("repo_name"),
}
if len(ctx.String("units")) == 0 {
opts.Wiki = true
opts.Issues = true
opts.Milestones = true
opts.Labels = true
opts.Releases = true
opts.Comments = true
opts.PullRequests = true
opts.ReleaseAssets = true
} else {
units := strings.Split(ctx.String("units"), ",")
for _, unit := range units {
switch strings.ToLower(unit) {
case "wiki":
opts.Wiki = true
case "issues":
opts.Issues = true
case "milestones":
opts.Milestones = true
case "labels":
opts.Labels = true
case "releases":
opts.Releases = true
case "release_assets":
opts.ReleaseAssets = true
case "comments":
opts.Comments = true
case "pull_requests":
opts.PullRequests = true
}
}
}
if err := migrations.RestoreRepository(
context.Background(),
ctx.String("repo_dir"), ctx.String("repo_dir"),
ctx.String("owner_name"), ctx.String("owner_name"),
ctx.String("repo_name"), ctx.String("repo_name"),
); err != nil { ctx.StringSlice("units"),
log.Fatal("Failed to restore repository: %v", err) )
return err if statusCode == http.StatusOK {
}
return nil return nil
} }
log.Fatal("Failed to restore repository: %v", errStr)
return errors.New(errStr)
}

View File

@ -13,6 +13,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings"
"time" "time"
"code.gitea.io/gitea/models" "code.gitea.io/gitea/models"
@ -563,8 +564,42 @@ func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.Mi
return nil return nil
} }
func updateOptionsUnits(opts *base.MigrateOptions, units []string) {
if len(units) == 0 {
opts.Wiki = true
opts.Issues = true
opts.Milestones = true
opts.Labels = true
opts.Releases = true
opts.Comments = true
opts.PullRequests = true
opts.ReleaseAssets = true
} else {
for _, unit := range units {
switch strings.ToLower(unit) {
case "wiki":
opts.Wiki = true
case "issues":
opts.Issues = true
case "milestones":
opts.Milestones = true
case "labels":
opts.Labels = true
case "releases":
opts.Releases = true
case "release_assets":
opts.ReleaseAssets = true
case "comments":
opts.Comments = true
case "pull_requests":
opts.PullRequests = true
}
}
}
}
// RestoreRepository restore a repository from the disk directory // RestoreRepository restore a repository from the disk directory
func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName string) error { func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName string, units []string) error {
doer, err := models.GetAdminUser() doer, err := models.GetAdminUser()
if err != nil { if err != nil {
return err return err
@ -580,17 +615,12 @@ func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName
} }
tp, _ := strconv.Atoi(opts["service_type"]) tp, _ := strconv.Atoi(opts["service_type"])
if err = migrateRepository(downloader, uploader, base.MigrateOptions{ var migrateOpts = base.MigrateOptions{
Wiki: true,
Issues: true,
Milestones: true,
Labels: true,
Releases: true,
Comments: true,
PullRequests: true,
ReleaseAssets: true,
GitServiceType: structs.GitServiceType(tp), GitServiceType: structs.GitServiceType(tp),
}); err != nil { }
updateOptionsUnits(&migrateOpts, units)
if err = migrateRepository(downloader, uploader, migrateOpts); err != nil {
if err1 := uploader.Rollback(); err1 != nil { if err1 := uploader.Rollback(); err1 != nil {
log.Error("rollback failed: %v", err1) log.Error("rollback failed: %v", err1)
} }

View File

@ -0,0 +1,60 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package private
import (
"fmt"
"io/ioutil"
"net/http"
"time"
"code.gitea.io/gitea/modules/setting"
jsoniter "github.com/json-iterator/go"
)
// RestoreParams structure holds a data for restore repository
type RestoreParams struct {
RepoDir string
OwnerName string
RepoName string
Units []string
}
// RestoreRepo calls the internal RestoreRepo function
func RestoreRepo(repoDir, ownerName, repoName string, units []string) (int, string) {
reqURL := setting.LocalURL + "api/internal/restore_repo"
req := newInternalRequest(reqURL, "POST")
req.SetTimeout(3*time.Second, 0) // since the request will spend much time, don't timeout
req = req.Header("Content-Type", "application/json")
json := jsoniter.ConfigCompatibleWithStandardLibrary
jsonBytes, _ := json.Marshal(RestoreParams{
RepoDir: repoDir,
OwnerName: ownerName,
RepoName: repoName,
Units: units,
})
req.Body(jsonBytes)
resp, err := req.Response()
if err != nil {
return http.StatusInternalServerError, fmt.Sprintf("Unable to contact gitea: %v, could you confirm it's running?", err.Error())
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
var ret = struct {
Err string `json:"err"`
}{}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return http.StatusInternalServerError, fmt.Sprintf("Response body error: %v", err.Error())
}
if err := json.Unmarshal(body, &ret); err != nil {
return http.StatusInternalServerError, fmt.Sprintf("Response body Unmarshal error: %v", err.Error())
}
}
return http.StatusOK, fmt.Sprintf("Restore repo %s/%s successfully", ownerName, repoName)
}

View File

@ -69,6 +69,7 @@ func Routes() *web.Route {
r.Post("/manager/add-logger", bind(private.LoggerOptions{}), AddLogger) r.Post("/manager/add-logger", bind(private.LoggerOptions{}), AddLogger)
r.Post("/manager/remove-logger/{group}/{name}", RemoveLogger) r.Post("/manager/remove-logger/{group}/{name}", RemoveLogger)
r.Post("/mail/send", SendEmail) r.Post("/mail/send", SendEmail)
r.Post("/restore_repo", RestoreRepo)
return r return r
} }

View File

@ -0,0 +1,51 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package private
import (
"io/ioutil"
myCtx "code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/migrations"
jsoniter "github.com/json-iterator/go"
)
// RestoreRepo restore a repository from data
func RestoreRepo(ctx *myCtx.PrivateContext) {
json := jsoniter.ConfigCompatibleWithStandardLibrary
bs, err := ioutil.ReadAll(ctx.Req.Body)
if err != nil {
ctx.JSON(500, map[string]string{
"err": err.Error(),
})
return
}
var params = struct {
RepoDir string
OwnerName string
RepoName string
Units []string
}{}
if err = json.Unmarshal(bs, &params); err != nil {
ctx.JSON(500, map[string]string{
"err": err.Error(),
})
return
}
if err := migrations.RestoreRepository(
ctx.Req.Context(),
params.RepoDir,
params.OwnerName,
params.RepoName,
params.Units,
); err != nil {
ctx.JSON(500, map[string]string{
"err": err.Error(),
})
} else {
ctx.Status(200)
}
}