Add pure SSH LFS support (#31516)

Fixes #17554
/claim #17554

Docs PR https://gitea.com/gitea/docs/pulls/49

To test, run pushes like: `GIT_TRACE=1` git push. The trace output
should mention "pure SSH connection".
This commit is contained in:
ConcurrentCrab 2024-09-27 19:57:37 +05:30 committed by GitHub
parent fdb1df9eca
commit 8a9fd7f771
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 945 additions and 53 deletions

View File

@ -344,6 +344,11 @@
"path": "github.com/cespare/xxhash/v2/LICENSE.txt",
"licenseText": "Copyright (c) 2016 Caleb Spare\n\nMIT License\n\nPermission is hereby granted, free of charge, to any person obtaining\na copy of this software and associated documentation files (the\n\"Software\"), to deal in the Software without restriction, including\nwithout limitation the rights to use, copy, modify, merge, publish,\ndistribute, sublicense, and/or sell copies of the Software, and to\npermit persons to whom the Software is furnished to do so, subject to\nthe following conditions:\n\nThe above copyright notice and this permission notice shall be\nincluded in all copies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND,\nEXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF\nMERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND\nNONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE\nLIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION\nOF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION\nWITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\n"
},
{
"name": "github.com/charmbracelet/git-lfs-transfer/transfer",
"path": "github.com/charmbracelet/git-lfs-transfer/transfer/LICENSE",
"licenseText": "MIT License\n\nCopyright (c) 2022-2023 Charmbracelet, Inc\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n"
},
{
"name": "github.com/chi-middleware/proxy",
"path": "github.com/chi-middleware/proxy/LICENSE",
@ -464,6 +469,11 @@
"path": "github.com/fxamacker/cbor/v2/LICENSE",
"licenseText": "MIT License\n\nCopyright (c) 2019-present Faye Amacker\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE."
},
{
"name": "github.com/git-lfs/pktline",
"path": "github.com/git-lfs/pktline/LICENSE.md",
"licenseText": "MIT License\n\nCopyright (c) 2014- GitHub, Inc. and Git LFS contributors\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n\nNote that Git LFS uses components from other Go modules (included in `vendor/`)\nwhich are under different licenses. See those LICENSE files for details.\n"
},
{
"name": "github.com/gliderlabs/ssh",
"path": "github.com/gliderlabs/ssh/LICENSE",

View File

@ -20,8 +20,10 @@ import (
asymkey_model "code.gitea.io/gitea/models/asymkey"
git_model "code.gitea.io/gitea/models/git"
"code.gitea.io/gitea/models/perm"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/lfstransfer"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/pprof"
"code.gitea.io/gitea/modules/private"
@ -36,7 +38,11 @@ import (
)
const (
lfsAuthenticateVerb = "git-lfs-authenticate"
verbUploadPack = "git-upload-pack"
verbUploadArchive = "git-upload-archive"
verbReceivePack = "git-receive-pack"
verbLfsAuthenticate = "git-lfs-authenticate"
verbLfsTransfer = "git-lfs-transfer"
)
// CmdServ represents the available serv sub-command.
@ -73,12 +79,18 @@ func setup(ctx context.Context, debug bool) {
}
var (
allowedCommands = map[string]perm.AccessMode{
"git-upload-pack": perm.AccessModeRead,
"git-upload-archive": perm.AccessModeRead,
"git-receive-pack": perm.AccessModeWrite,
lfsAuthenticateVerb: perm.AccessModeNone,
}
// keep getAccessMode() in sync
allowedCommands = container.SetOf(
verbUploadPack,
verbUploadArchive,
verbReceivePack,
verbLfsAuthenticate,
verbLfsTransfer,
)
allowedCommandsLfs = container.SetOf(
verbLfsAuthenticate,
verbLfsTransfer,
)
alphaDashDotPattern = regexp.MustCompile(`[^\w-\.]`)
)
@ -124,6 +136,45 @@ func handleCliResponseExtra(extra private.ResponseExtra) error {
return nil
}
func getAccessMode(verb, lfsVerb string) perm.AccessMode {
switch verb {
case verbUploadPack, verbUploadArchive:
return perm.AccessModeRead
case verbReceivePack:
return perm.AccessModeWrite
case verbLfsAuthenticate, verbLfsTransfer:
switch lfsVerb {
case "upload":
return perm.AccessModeWrite
case "download":
return perm.AccessModeRead
}
}
// should be unreachable
return perm.AccessModeNone
}
func getLFSAuthToken(ctx context.Context, lfsVerb string, results *private.ServCommandResults) (string, error) {
now := time.Now()
claims := lfs.Claims{
RegisteredClaims: jwt.RegisteredClaims{
ExpiresAt: jwt.NewNumericDate(now.Add(setting.LFS.HTTPAuthExpiry)),
NotBefore: jwt.NewNumericDate(now),
},
RepoID: results.RepoID,
Op: lfsVerb,
UserID: results.UserID,
}
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
// Sign and get the complete encoded token as a string using the secret
tokenString, err := token.SignedString(setting.LFS.JWTSecretBytes)
if err != nil {
return "", fail(ctx, "Failed to sign JWT Token", "Failed to sign JWT token: %v", err)
}
return fmt.Sprintf("Bearer %s", tokenString), nil
}
func runServ(c *cli.Context) error {
ctx, cancel := installSignals()
defer cancel()
@ -198,15 +249,6 @@ func runServ(c *cli.Context) error {
repoPath := strings.TrimPrefix(words[1], "/")
var lfsVerb string
if verb == lfsAuthenticateVerb {
if !setting.LFS.StartServer {
return fail(ctx, "Unknown git command", "LFS authentication request over SSH denied, LFS support is disabled")
}
if len(words) > 2 {
lfsVerb = words[2]
}
}
rr := strings.SplitN(repoPath, "/", 2)
if len(rr) != 2 {
@ -243,53 +285,52 @@ func runServ(c *cli.Context) error {
}()
}
requestedMode, has := allowedCommands[verb]
if !has {
if allowedCommands.Contains(verb) {
if allowedCommandsLfs.Contains(verb) {
if !setting.LFS.StartServer {
return fail(ctx, "Unknown git command", "LFS authentication request over SSH denied, LFS support is disabled")
}
if verb == verbLfsTransfer && !setting.LFS.AllowPureSSH {
return fail(ctx, "Unknown git command", "LFS SSH transfer connection denied, pure SSH protocol is disabled")
}
if len(words) > 2 {
lfsVerb = words[2]
}
}
} else {
return fail(ctx, "Unknown git command", "Unknown git command %s", verb)
}
if verb == lfsAuthenticateVerb {
if lfsVerb == "upload" {
requestedMode = perm.AccessModeWrite
} else if lfsVerb == "download" {
requestedMode = perm.AccessModeRead
} else {
return fail(ctx, "Unknown LFS verb", "Unknown lfs verb %s", lfsVerb)
}
}
requestedMode := getAccessMode(verb, lfsVerb)
results, extra := private.ServCommand(ctx, keyID, username, reponame, requestedMode, verb, lfsVerb)
if extra.HasError() {
return fail(ctx, extra.UserMsg, "ServCommand failed: %s", extra.Error)
}
// LFS SSH protocol
if verb == verbLfsTransfer {
token, err := getLFSAuthToken(ctx, lfsVerb, results)
if err != nil {
return err
}
return lfstransfer.Main(ctx, repoPath, lfsVerb, token)
}
// LFS token authentication
if verb == lfsAuthenticateVerb {
if verb == verbLfsAuthenticate {
url := fmt.Sprintf("%s%s/%s.git/info/lfs", setting.AppURL, url.PathEscape(results.OwnerName), url.PathEscape(results.RepoName))
now := time.Now()
claims := lfs.Claims{
RegisteredClaims: jwt.RegisteredClaims{
ExpiresAt: jwt.NewNumericDate(now.Add(setting.LFS.HTTPAuthExpiry)),
NotBefore: jwt.NewNumericDate(now),
},
RepoID: results.RepoID,
Op: lfsVerb,
UserID: results.UserID,
}
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
// Sign and get the complete encoded token as a string using the secret
tokenString, err := token.SignedString(setting.LFS.JWTSecretBytes)
token, err := getLFSAuthToken(ctx, lfsVerb, results)
if err != nil {
return fail(ctx, "Failed to sign JWT Token", "Failed to sign JWT token: %v", err)
return err
}
tokenAuthentication := &git_model.LFSTokenResponse{
Header: make(map[string]string),
Href: url,
}
tokenAuthentication.Header["Authorization"] = fmt.Sprintf("Bearer %s", tokenString)
tokenAuthentication.Header["Authorization"] = token
enc := json.NewEncoder(os.Stdout)
err = enc.Encode(tokenAuthentication)

View File

@ -306,6 +306,8 @@ RUN_USER = ; git
;; Enables git-lfs support. true or false, default is false.
;LFS_START_SERVER = false
;;
;; Enables git-lfs SSH protocol support. true or false, default is false.
;LFS_ALLOW_PURE_SSH = false
;;
;; LFS authentication secret, change this yourself
;LFS_JWT_SECRET =

4
go.mod
View File

@ -35,6 +35,7 @@ require (
github.com/blevesearch/bleve/v2 v2.4.2
github.com/buildkite/terminal-to-html/v3 v3.12.1
github.com/caddyserver/certmagic v0.21.3
github.com/charmbracelet/git-lfs-transfer v0.2.0
github.com/chi-middleware/proxy v1.1.1
github.com/dimiro1/reply v0.0.0-20200315094148-d0136a4c9e21
github.com/djherbis/buffer v1.2.0
@ -197,6 +198,7 @@ require (
github.com/fatih/color v1.17.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/fxamacker/cbor/v2 v2.6.0 // indirect
github.com/git-lfs/pktline v0.0.0-20230103162542-ca444d533ef1 // indirect
github.com/go-ap/errors v0.0.0-20240304112515-6077fa9c17b0 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
github.com/go-enry/go-oniguruma v1.2.1 // indirect
@ -329,6 +331,8 @@ replace github.com/shurcooL/vfsgen => github.com/lunny/vfsgen v0.0.0-20220105142
replace github.com/nektos/act => gitea.com/gitea/act v0.259.1
replace github.com/charmbracelet/git-lfs-transfer => gitea.com/gitea/git-lfs-transfer v0.2.0
// TODO: This could be removed after https://github.com/mholt/archiver/pull/396 merged
replace github.com/mholt/archiver/v3 => github.com/anchore/archiver/v3 v3.5.2

4
go.sum
View File

@ -18,6 +18,8 @@ git.sr.ht/~mariusor/go-xsd-duration v0.0.0-20220703122237-02e73435a078 h1:cliQ4H
git.sr.ht/~mariusor/go-xsd-duration v0.0.0-20220703122237-02e73435a078/go.mod h1:g/V2Hjas6Z1UHUp4yIx6bATpNzJ7DYtD0FG3+xARWxs=
gitea.com/gitea/act v0.259.1 h1:8GG1o/xtUHl3qjn5f0h/2FXrT5ubBn05TJOM5ry+FBw=
gitea.com/gitea/act v0.259.1/go.mod h1:UxZWRYqQG2Yj4+4OqfGWW5a3HELwejyWFQyU7F1jUD8=
gitea.com/gitea/git-lfs-transfer v0.2.0 h1:baHaNoBSRaeq/xKayEXwiDQtlIjps4Ac/Ll4KqLMB40=
gitea.com/gitea/git-lfs-transfer v0.2.0/go.mod h1:UrXUCm3xLQkq15fu7qlXHUMlrhdlXHoi13KH2Dfiits=
gitea.com/go-chi/binding v0.0.0-20240430071103-39a851e106ed h1:EZZBtilMLSZNWtHHcgq2mt6NSGhJSZBuduAlinMEmso=
gitea.com/go-chi/binding v0.0.0-20240430071103-39a851e106ed/go.mod h1:E3i3cgB04dDx0v3CytCgRTTn9Z/9x891aet3r456RVw=
gitea.com/go-chi/cache v0.2.1 h1:bfAPkvXlbcZxPCpcmDVCWoHgiBSBmZN/QosnZvEC0+g=
@ -291,6 +293,8 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/fxamacker/cbor/v2 v2.6.0 h1:sU6J2usfADwWlYDAFhZBQ6TnLFBHxgesMrQfQgk1tWA=
github.com/fxamacker/cbor/v2 v2.6.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
github.com/git-lfs/pktline v0.0.0-20230103162542-ca444d533ef1 h1:mtDjlmloH7ytdblogrMz1/8Hqua1y8B4ID+bh3rvod0=
github.com/git-lfs/pktline v0.0.0-20230103162542-ca444d533ef1/go.mod h1:fenKRzpXDjNpsIBhuhUzvjCKlDjKam0boRAenTE0Q6A=
github.com/gliderlabs/ssh v0.3.7 h1:iV3Bqi942d9huXnzEF2Mt+CY9gLu8DNM4Obd+8bODRE=
github.com/gliderlabs/ssh v0.3.7/go.mod h1:zpHEXBstFnQYtGnB8k8kQLol82umzn/2/snG7alWVD8=
github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=

View File

@ -0,0 +1,301 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package backend
import (
"bytes"
"context"
"encoding/base64"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/setting"
"github.com/charmbracelet/git-lfs-transfer/transfer"
)
// Version is the git-lfs-transfer protocol version number.
const Version = "1"
// Capabilities is a list of Git LFS capabilities supported by this package.
var Capabilities = []string{
"version=" + Version,
"locking",
}
var _ transfer.Backend = &GiteaBackend{}
// GiteaBackend is an adapter between git-lfs-transfer library and Gitea's internal LFS API
type GiteaBackend struct {
ctx context.Context
server *url.URL
op string
token string
itoken string
logger transfer.Logger
}
func New(ctx context.Context, repo, op, token string, logger transfer.Logger) (transfer.Backend, error) {
// runServ guarantees repo will be in form [owner]/[name].git
server, err := url.Parse(setting.LocalURL)
if err != nil {
return nil, err
}
server = server.JoinPath("api/internal/repo", repo, "info/lfs")
return &GiteaBackend{ctx: ctx, server: server, op: op, token: token, itoken: fmt.Sprintf("Bearer %s", setting.InternalToken), logger: logger}, nil
}
// Batch implements transfer.Backend
func (g *GiteaBackend) Batch(_ string, pointers []transfer.BatchItem, args transfer.Args) ([]transfer.BatchItem, error) {
reqBody := lfs.BatchRequest{Operation: g.op}
if transfer, ok := args[argTransfer]; ok {
reqBody.Transfers = []string{transfer}
}
if ref, ok := args[argRefname]; ok {
reqBody.Ref = &lfs.Reference{Name: ref}
}
reqBody.Objects = make([]lfs.Pointer, len(pointers))
for i := range pointers {
reqBody.Objects[i].Oid = pointers[i].Oid
reqBody.Objects[i].Size = pointers[i].Size
}
bodyBytes, err := json.Marshal(reqBody)
if err != nil {
g.logger.Log("json marshal error", err)
return nil, err
}
url := g.server.JoinPath("objects/batch").String()
headers := map[string]string{
headerAuthorisation: g.itoken,
headerAuthX: g.token,
headerAccept: mimeGitLFS,
headerContentType: mimeGitLFS,
}
req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes)
resp, err := req.Response()
if err != nil {
g.logger.Log("http request error", err)
return nil, err
}
if resp.StatusCode != http.StatusOK {
g.logger.Log("http statuscode error", resp.StatusCode, statusCodeToErr(resp.StatusCode))
return nil, statusCodeToErr(resp.StatusCode)
}
defer resp.Body.Close()
respBytes, err := io.ReadAll(resp.Body)
if err != nil {
g.logger.Log("http read error", err)
return nil, err
}
var respBody lfs.BatchResponse
err = json.Unmarshal(respBytes, &respBody)
if err != nil {
g.logger.Log("json umarshal error", err)
return nil, err
}
// rebuild slice, we can't rely on order in resp being the same as req
pointers = pointers[:0]
opNum := opMap[g.op]
for _, obj := range respBody.Objects {
pointer := transfer.Pointer{Oid: obj.Pointer.Oid, Size: obj.Pointer.Size}
item := transfer.BatchItem{Pointer: pointer, Args: map[string]string{}}
switch opNum {
case opDownload:
if action, ok := obj.Actions[actionDownload]; ok {
item.Present = true
idMap := obj.Actions
idMapBytes, err := json.Marshal(idMap)
if err != nil {
g.logger.Log("json marshal error", err)
return nil, err
}
idMapStr := base64.StdEncoding.EncodeToString(idMapBytes)
item.Args[argID] = idMapStr
if authHeader, ok := action.Header[headerAuthorisation]; ok {
authHeaderB64 := base64.StdEncoding.EncodeToString([]byte(authHeader))
item.Args[argToken] = authHeaderB64
}
if action.ExpiresAt != nil {
item.Args[argExpiresAt] = action.ExpiresAt.String()
}
} else {
// must be an error, but the SSH protocol can't propagate individual errors
g.logger.Log("object not found", obj.Pointer.Oid, obj.Pointer.Size)
item.Present = false
}
case opUpload:
if action, ok := obj.Actions[actionUpload]; ok {
item.Present = false
idMap := obj.Actions
idMapBytes, err := json.Marshal(idMap)
if err != nil {
g.logger.Log("json marshal error", err)
return nil, err
}
idMapStr := base64.StdEncoding.EncodeToString(idMapBytes)
item.Args[argID] = idMapStr
if authHeader, ok := action.Header[headerAuthorisation]; ok {
authHeaderB64 := base64.StdEncoding.EncodeToString([]byte(authHeader))
item.Args[argToken] = authHeaderB64
}
if action.ExpiresAt != nil {
item.Args[argExpiresAt] = action.ExpiresAt.String()
}
} else {
item.Present = true
}
}
pointers = append(pointers, item)
}
return pointers, nil
}
// Download implements transfer.Backend. The returned reader must be closed by the
// caller.
func (g *GiteaBackend) Download(oid string, args transfer.Args) (io.ReadCloser, int64, error) {
idMapStr, exists := args[argID]
if !exists {
return nil, 0, ErrMissingID
}
idMapBytes, err := base64.StdEncoding.DecodeString(idMapStr)
if err != nil {
g.logger.Log("base64 decode error", err)
return nil, 0, transfer.ErrCorruptData
}
idMap := map[string]*lfs.Link{}
err = json.Unmarshal(idMapBytes, &idMap)
if err != nil {
g.logger.Log("json unmarshal error", err)
return nil, 0, transfer.ErrCorruptData
}
action, exists := idMap[actionDownload]
if !exists {
g.logger.Log("argument id incorrect")
return nil, 0, transfer.ErrCorruptData
}
url := action.Href
headers := map[string]string{
headerAuthorisation: g.itoken,
headerAuthX: g.token,
headerAccept: mimeOctetStream,
}
req := newInternalRequest(g.ctx, url, http.MethodGet, headers, nil)
resp, err := req.Response()
if err != nil {
return nil, 0, err
}
if resp.StatusCode != http.StatusOK {
return nil, 0, statusCodeToErr(resp.StatusCode)
}
defer resp.Body.Close()
respBytes, err := io.ReadAll(resp.Body)
if err != nil {
return nil, 0, err
}
respSize := int64(len(respBytes))
respBuf := io.NopCloser(bytes.NewBuffer(respBytes))
return respBuf, respSize, nil
}
// StartUpload implements transfer.Backend.
func (g *GiteaBackend) Upload(oid string, size int64, r io.Reader, args transfer.Args) error {
idMapStr, exists := args[argID]
if !exists {
return ErrMissingID
}
idMapBytes, err := base64.StdEncoding.DecodeString(idMapStr)
if err != nil {
g.logger.Log("base64 decode error", err)
return transfer.ErrCorruptData
}
idMap := map[string]*lfs.Link{}
err = json.Unmarshal(idMapBytes, &idMap)
if err != nil {
g.logger.Log("json unmarshal error", err)
return transfer.ErrCorruptData
}
action, exists := idMap[actionUpload]
if !exists {
g.logger.Log("argument id incorrect")
return transfer.ErrCorruptData
}
url := action.Href
headers := map[string]string{
headerAuthorisation: g.itoken,
headerAuthX: g.token,
headerContentType: mimeOctetStream,
headerContentLength: strconv.FormatInt(size, 10),
}
reqBytes, err := io.ReadAll(r)
if err != nil {
return err
}
req := newInternalRequest(g.ctx, url, http.MethodPut, headers, reqBytes)
resp, err := req.Response()
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return statusCodeToErr(resp.StatusCode)
}
return nil
}
// Verify implements transfer.Backend.
func (g *GiteaBackend) Verify(oid string, size int64, args transfer.Args) (transfer.Status, error) {
reqBody := lfs.Pointer{Oid: oid, Size: size}
bodyBytes, err := json.Marshal(reqBody)
if err != nil {
return transfer.NewStatus(transfer.StatusInternalServerError), err
}
idMapStr, exists := args[argID]
if !exists {
return transfer.NewStatus(transfer.StatusBadRequest, "missing argument: id"), ErrMissingID
}
idMapBytes, err := base64.StdEncoding.DecodeString(idMapStr)
if err != nil {
g.logger.Log("base64 decode error", err)
return transfer.NewStatus(transfer.StatusBadRequest, "corrupt argument: id"), transfer.ErrCorruptData
}
idMap := map[string]*lfs.Link{}
err = json.Unmarshal(idMapBytes, &idMap)
if err != nil {
g.logger.Log("json unmarshal error", err)
return transfer.NewStatus(transfer.StatusBadRequest, "corrupt argument: id"), transfer.ErrCorruptData
}
action, exists := idMap[actionVerify]
if !exists {
// the server sent no verify action
return transfer.SuccessStatus(), nil
}
url := action.Href
headers := map[string]string{
headerAuthorisation: g.itoken,
headerAuthX: g.token,
headerAccept: mimeGitLFS,
headerContentType: mimeGitLFS,
}
req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes)
resp, err := req.Response()
if err != nil {
return transfer.NewStatus(transfer.StatusInternalServerError), err
}
if resp.StatusCode != http.StatusOK {
return transfer.NewStatus(uint32(resp.StatusCode), http.StatusText(resp.StatusCode)), statusCodeToErr(resp.StatusCode)
}
return transfer.SuccessStatus(), nil
}
// LockBackend implements transfer.Backend.
func (g *GiteaBackend) LockBackend(_ transfer.Args) transfer.LockBackend {
return newGiteaLockBackend(g)
}

View File

@ -0,0 +1,296 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package backend
import (
"context"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"time"
"code.gitea.io/gitea/modules/json"
lfslock "code.gitea.io/gitea/modules/structs"
"github.com/charmbracelet/git-lfs-transfer/transfer"
)
var _ transfer.LockBackend = &giteaLockBackend{}
type giteaLockBackend struct {
ctx context.Context
g *GiteaBackend
server *url.URL
token string
itoken string
logger transfer.Logger
}
func newGiteaLockBackend(g *GiteaBackend) transfer.LockBackend {
server := g.server.JoinPath("locks")
return &giteaLockBackend{ctx: g.ctx, g: g, server: server, token: g.token, itoken: g.itoken, logger: g.logger}
}
// Create implements transfer.LockBackend
func (g *giteaLockBackend) Create(path, refname string) (transfer.Lock, error) {
reqBody := lfslock.LFSLockRequest{Path: path}
bodyBytes, err := json.Marshal(reqBody)
if err != nil {
g.logger.Log("json marshal error", err)
return nil, err
}
url := g.server.String()
headers := map[string]string{
headerAuthorisation: g.itoken,
headerAuthX: g.token,
headerAccept: mimeGitLFS,
headerContentType: mimeGitLFS,
}
req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes)
resp, err := req.Response()
if err != nil {
g.logger.Log("http request error", err)
return nil, err
}
defer resp.Body.Close()
respBytes, err := io.ReadAll(resp.Body)
if err != nil {
g.logger.Log("http read error", err)
return nil, err
}
if resp.StatusCode != http.StatusCreated {
g.logger.Log("http statuscode error", resp.StatusCode, statusCodeToErr(resp.StatusCode))
return nil, statusCodeToErr(resp.StatusCode)
}
var respBody lfslock.LFSLockResponse
err = json.Unmarshal(respBytes, &respBody)
if err != nil {
g.logger.Log("json umarshal error", err)
return nil, err
}
if respBody.Lock == nil {
g.logger.Log("api returned nil lock")
return nil, fmt.Errorf("api returned nil lock")
}
respLock := respBody.Lock
owner := userUnknown
if respLock.Owner != nil {
owner = respLock.Owner.Name
}
lock := newGiteaLock(g, respLock.ID, respLock.Path, respLock.LockedAt, owner)
return lock, nil
}
// Unlock implements transfer.LockBackend
func (g *giteaLockBackend) Unlock(lock transfer.Lock) error {
reqBody := lfslock.LFSLockDeleteRequest{}
bodyBytes, err := json.Marshal(reqBody)
if err != nil {
g.logger.Log("json marshal error", err)
return err
}
url := g.server.JoinPath(lock.ID(), "unlock").String()
headers := map[string]string{
headerAuthorisation: g.itoken,
headerAuthX: g.token,
headerAccept: mimeGitLFS,
headerContentType: mimeGitLFS,
}
req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes)
resp, err := req.Response()
if err != nil {
g.logger.Log("http request error", err)
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
g.logger.Log("http statuscode error", resp.StatusCode, statusCodeToErr(resp.StatusCode))
return statusCodeToErr(resp.StatusCode)
}
// no need to read response
return nil
}
// FromPath implements transfer.LockBackend
func (g *giteaLockBackend) FromPath(path string) (transfer.Lock, error) {
v := url.Values{
argPath: []string{path},
}
respLocks, _, err := g.queryLocks(v)
if err != nil {
return nil, err
}
if len(respLocks) == 0 {
return nil, transfer.ErrNotFound
}
return respLocks[0], nil
}
// FromID implements transfer.LockBackend
func (g *giteaLockBackend) FromID(id string) (transfer.Lock, error) {
v := url.Values{
argID: []string{id},
}
respLocks, _, err := g.queryLocks(v)
if err != nil {
return nil, err
}
if len(respLocks) == 0 {
return nil, transfer.ErrNotFound
}
return respLocks[0], nil
}
// Range implements transfer.LockBackend
func (g *giteaLockBackend) Range(cursor string, limit int, iter func(transfer.Lock) error) (string, error) {
v := url.Values{
argLimit: []string{strconv.FormatInt(int64(limit), 10)},
}
if cursor != "" {
v[argCursor] = []string{cursor}
}
respLocks, cursor, err := g.queryLocks(v)
if err != nil {
return "", err
}
for _, lock := range respLocks {
err := iter(lock)
if err != nil {
return "", err
}
}
return cursor, nil
}
func (g *giteaLockBackend) queryLocks(v url.Values) ([]transfer.Lock, string, error) {
urlq := g.server.JoinPath() // get a copy
urlq.RawQuery = v.Encode()
url := urlq.String()
headers := map[string]string{
headerAuthorisation: g.itoken,
headerAuthX: g.token,
headerAccept: mimeGitLFS,
headerContentType: mimeGitLFS,
}
req := newInternalRequest(g.ctx, url, http.MethodGet, headers, nil)
resp, err := req.Response()
if err != nil {
g.logger.Log("http request error", err)
return nil, "", err
}
defer resp.Body.Close()
respBytes, err := io.ReadAll(resp.Body)
if err != nil {
g.logger.Log("http read error", err)
return nil, "", err
}
if resp.StatusCode != http.StatusOK {
g.logger.Log("http statuscode error", resp.StatusCode, statusCodeToErr(resp.StatusCode))
return nil, "", statusCodeToErr(resp.StatusCode)
}
var respBody lfslock.LFSLockList
err = json.Unmarshal(respBytes, &respBody)
if err != nil {
g.logger.Log("json umarshal error", err)
return nil, "", err
}
respLocks := make([]transfer.Lock, 0, len(respBody.Locks))
for _, respLock := range respBody.Locks {
owner := userUnknown
if respLock.Owner != nil {
owner = respLock.Owner.Name
}
lock := newGiteaLock(g, respLock.ID, respLock.Path, respLock.LockedAt, owner)
respLocks = append(respLocks, lock)
}
return respLocks, respBody.Next, nil
}
var _ transfer.Lock = &giteaLock{}
type giteaLock struct {
g *giteaLockBackend
id string
path string
lockedAt time.Time
owner string
}
func newGiteaLock(g *giteaLockBackend, id, path string, lockedAt time.Time, owner string) transfer.Lock {
return &giteaLock{g: g, id: id, path: path, lockedAt: lockedAt, owner: owner}
}
// Unlock implements transfer.Lock
func (g *giteaLock) Unlock() error {
return g.g.Unlock(g)
}
// ID implements transfer.Lock
func (g *giteaLock) ID() string {
return g.id
}
// Path implements transfer.Lock
func (g *giteaLock) Path() string {
return g.path
}
// FormattedTimestamp implements transfer.Lock
func (g *giteaLock) FormattedTimestamp() string {
return g.lockedAt.UTC().Format(time.RFC3339)
}
// OwnerName implements transfer.Lock
func (g *giteaLock) OwnerName() string {
return g.owner
}
func (g *giteaLock) CurrentUser() (string, error) {
return userSelf, nil
}
// AsLockSpec implements transfer.Lock
func (g *giteaLock) AsLockSpec(ownerID bool) ([]string, error) {
msgs := []string{
fmt.Sprintf("lock %s", g.ID()),
fmt.Sprintf("path %s %s", g.ID(), g.Path()),
fmt.Sprintf("locked-at %s %s", g.ID(), g.FormattedTimestamp()),
fmt.Sprintf("ownername %s %s", g.ID(), g.OwnerName()),
}
if ownerID {
user, err := g.CurrentUser()
if err != nil {
return nil, fmt.Errorf("error getting current user: %w", err)
}
who := "theirs"
if user == g.OwnerName() {
who = "ours"
}
msgs = append(msgs, fmt.Sprintf("owner %s %s", g.ID(), who))
}
return msgs, nil
}
// AsArguments implements transfer.Lock
func (g *giteaLock) AsArguments() []string {
return []string{
fmt.Sprintf("id=%s", g.ID()),
fmt.Sprintf("path=%s", g.Path()),
fmt.Sprintf("locked-at=%s", g.FormattedTimestamp()),
fmt.Sprintf("ownername=%s", g.OwnerName()),
}
}

View File

@ -0,0 +1,141 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package backend
import (
"context"
"crypto/tls"
"fmt"
"net"
"net/http"
"time"
"code.gitea.io/gitea/modules/httplib"
"code.gitea.io/gitea/modules/proxyprotocol"
"code.gitea.io/gitea/modules/setting"
"github.com/charmbracelet/git-lfs-transfer/transfer"
)
// HTTP headers
const (
headerAccept = "Accept"
headerAuthorisation = "Authorization"
headerAuthX = "X-Auth"
headerContentType = "Content-Type"
headerContentLength = "Content-Length"
)
// MIME types
const (
mimeGitLFS = "application/vnd.git-lfs+json"
mimeOctetStream = "application/octet-stream"
)
// SSH protocol action keys
const (
actionDownload = "download"
actionUpload = "upload"
actionVerify = "verify"
)
// SSH protocol argument keys
const (
argCursor = "cursor"
argExpiresAt = "expires-at"
argID = "id"
argLimit = "limit"
argPath = "path"
argRefname = "refname"
argToken = "token"
argTransfer = "transfer"
)
// Default username constants
const (
userSelf = "(self)"
userUnknown = "(unknown)"
)
// Operations enum
const (
opNone = iota
opDownload
opUpload
)
var opMap = map[string]int{
"download": opDownload,
"upload": opUpload,
}
var ErrMissingID = fmt.Errorf("%w: missing id arg", transfer.ErrMissingData)
func statusCodeToErr(code int) error {
switch code {
case http.StatusBadRequest:
return transfer.ErrParseError
case http.StatusConflict:
return transfer.ErrConflict
case http.StatusForbidden:
return transfer.ErrForbidden
case http.StatusNotFound:
return transfer.ErrNotFound
case http.StatusUnauthorized:
return transfer.ErrUnauthorized
default:
return fmt.Errorf("server returned status %v: %v", code, http.StatusText(code))
}
}
func newInternalRequest(ctx context.Context, url, method string, headers map[string]string, body []byte) *httplib.Request {
req := httplib.NewRequest(url, method).
SetContext(ctx).
SetTimeout(10*time.Second, 60*time.Second).
SetTLSClientConfig(&tls.Config{
InsecureSkipVerify: true,
})
if setting.Protocol == setting.HTTPUnix {
req.SetTransport(&http.Transport{
DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
var d net.Dialer
conn, err := d.DialContext(ctx, "unix", setting.HTTPAddr)
if err != nil {
return conn, err
}
if setting.LocalUseProxyProtocol {
if err = proxyprotocol.WriteLocalHeader(conn); err != nil {
_ = conn.Close()
return nil, err
}
}
return conn, err
},
})
} else if setting.LocalUseProxyProtocol {
req.SetTransport(&http.Transport{
DialContext: func(ctx context.Context, network, address string) (net.Conn, error) {
var d net.Dialer
conn, err := d.DialContext(ctx, network, address)
if err != nil {
return conn, err
}
if err = proxyprotocol.WriteLocalHeader(conn); err != nil {
_ = conn.Close()
return nil, err
}
return conn, err
},
})
}
for k, v := range headers {
req.Header(k, v)
}
req.Body(body)
return req
}

View File

@ -0,0 +1,21 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package lfstransfer
import (
"github.com/charmbracelet/git-lfs-transfer/transfer"
)
var _ transfer.Logger = (*GiteaLogger)(nil)
// noop logger for passing into transfer
type GiteaLogger struct{}
func newLogger() transfer.Logger {
return &GiteaLogger{}
}
// Log implements transfer.Logger
func (g *GiteaLogger) Log(msg string, itms ...any) {
}

View File

@ -0,0 +1,42 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package lfstransfer
import (
"context"
"fmt"
"os"
"code.gitea.io/gitea/modules/lfstransfer/backend"
"github.com/charmbracelet/git-lfs-transfer/transfer"
)
func Main(ctx context.Context, repo, verb, token string) error {
logger := newLogger()
pktline := transfer.NewPktline(os.Stdin, os.Stdout, logger)
giteaBackend, err := backend.New(ctx, repo, verb, token, logger)
if err != nil {
return err
}
for _, cap := range backend.Capabilities {
if err := pktline.WritePacketText(cap); err != nil {
logger.Log("error sending capability due to error:", err)
}
}
if err := pktline.WriteFlush(); err != nil {
logger.Log("error flushing capabilities:", err)
}
p := transfer.NewProcessor(pktline, giteaBackend, logger)
defer logger.Log("done processing commands")
switch verb {
case "upload":
return p.ProcessCommands(transfer.UploadOperation)
case "download":
return p.ProcessCommands(transfer.DownloadOperation)
default:
return fmt.Errorf("unknown operation %q", verb)
}
}

View File

@ -13,6 +13,7 @@ import (
// LFS represents the configuration for Git LFS
var LFS = struct {
StartServer bool `ini:"LFS_START_SERVER"`
AllowPureSSH bool `ini:"LFS_ALLOW_PURE_SSH"`
JWTSecretBytes []byte `ini:"-"`
HTTPAuthExpiry time.Duration `ini:"LFS_HTTP_AUTH_EXPIRY"`
MaxFileSize int64 `ini:"LFS_MAX_FILE_SIZE"`

View File

@ -12,7 +12,9 @@ import (
"code.gitea.io/gitea/modules/private"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/web"
"code.gitea.io/gitea/routers/common"
"code.gitea.io/gitea/services/context"
"code.gitea.io/gitea/services/lfs"
"gitea.com/go-chi/binding"
chi_middleware "github.com/go-chi/chi/v5/middleware"
@ -46,6 +48,14 @@ func bind[T any](_ T) any {
}
}
// SwapAuthToken swaps Authorization header with X-Auth header
func swapAuthToken(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
req.Header.Set("Authorization", req.Header.Get("X-Auth"))
next.ServeHTTP(w, req)
})
}
// Routes registers all internal APIs routes to web application.
// These APIs will be invoked by internal commands for example `gitea serv` and etc.
func Routes() *web.Router {
@ -80,5 +90,25 @@ func Routes() *web.Router {
r.Post("/restore_repo", RestoreRepo)
r.Post("/actions/generate_actions_runner_token", GenerateActionsRunnerToken)
r.Group("/repo/{username}/{reponame}", func() {
r.Group("/info/lfs", func() {
r.Post("/objects/batch", lfs.CheckAcceptMediaType, lfs.BatchHandler)
r.Put("/objects/{oid}/{size}", lfs.UploadHandler)
r.Get("/objects/{oid}/{filename}", lfs.DownloadHandler)
r.Get("/objects/{oid}", lfs.DownloadHandler)
r.Post("/verify", lfs.CheckAcceptMediaType, lfs.VerifyHandler)
r.Group("/locks", func() {
r.Get("/", lfs.GetListLockHandler)
r.Post("/", lfs.PostLockHandler)
r.Post("/verify", lfs.VerifyLockHandler)
r.Post("/{lid}/unlock", lfs.UnLockHandler)
}, lfs.CheckAcceptMediaType)
r.Any("/*", func(ctx *context.Context) {
ctx.NotFound("", nil)
})
}, swapAuthToken)
}, common.Sessioner(), context.Contexter())
// end "/repo/{username}/{reponame}": git (LFS) API mirror
return r
}

View File

@ -136,16 +136,15 @@ func ServCommand(ctx *context.PrivateContext) {
if err != nil {
if repo_model.IsErrRepoNotExist(err) {
repoExist = false
for _, verb := range ctx.FormStrings("verb") {
if verb == "git-upload-pack" {
// User is fetching/cloning a non-existent repository
log.Warn("Failed authentication attempt (cannot find repository: %s/%s) from %s", results.OwnerName, results.RepoName, ctx.RemoteAddr())
ctx.JSON(http.StatusNotFound, private.Response{
UserMsg: fmt.Sprintf("Cannot find repository: %s/%s", results.OwnerName, results.RepoName),
})
return
}
if mode == perm.AccessModeRead {
// User is fetching/cloning a non-existent repository
log.Warn("Failed authentication attempt (cannot find repository: %s/%s) from %s", results.OwnerName, results.RepoName, ctx.RemoteAddr())
ctx.JSON(http.StatusNotFound, private.Response{
UserMsg: fmt.Sprintf("Cannot find repository: %s/%s", results.OwnerName, results.RepoName),
})
return
}
// else fallthrough (push-to-create may kick in below)
} else {
log.Error("Unable to get repository: %s/%s Error: %v", results.OwnerName, results.RepoName, err)
ctx.JSON(http.StatusInternalServerError, private.Response{