gitea/services/repository/files/update.go
wxiaoguang 6bc3079c00
Refactor git command package to improve security and maintainability (#22678)
This PR follows #21535 (and replace #22592)

## Review without space diff

https://github.com/go-gitea/gitea/pull/22678/files?diff=split&w=1

## Purpose of this PR

1. Make git module command completely safe (risky user inputs won't be
passed as argument option anymore)
2. Avoid low-level mistakes like
https://github.com/go-gitea/gitea/pull/22098#discussion_r1045234918
3. Remove deprecated and dirty `CmdArgCheck` function, hide the `CmdArg`
type
4. Simplify code when using git command

## The main idea of this PR

* Move the `git.CmdArg` to the `internal` package, then no other package
except `git` could use it. Then developers could never do
`AddArguments(git.CmdArg(userInput))` any more.
* Introduce `git.ToTrustedCmdArgs`, it's for user-provided and already
trusted arguments. It's only used in a few cases, for example: use git
arguments from config file, help unit test with some arguments.
* Introduce `AddOptionValues` and `AddOptionFormat`, they make code more
clear and simple:
    * Before: `AddArguments("-m").AddDynamicArguments(message)`
    * After: `AddOptionValues("-m", message)`
    * -
* Before: `AddArguments(git.CmdArg(fmt.Sprintf("--author='%s <%s>'",
sig.Name, sig.Email)))`
* After: `AddOptionFormat("--author='%s <%s>'", sig.Name, sig.Email)`

## FAQ

### Why these changes were not done in #21535 ?

#21535 is mainly a search&replace, it did its best to not change too
much logic.

Making the framework better needs a lot of changes, so this separate PR
is needed as the second step.


### The naming of `AddOptionXxx`

According to git's manual, the `--xxx` part is called `option`.

### How can it guarantee that `internal.CmdArg` won't be not misused?

Go's specification guarantees that. Trying to access other package's
internal package causes compilation error.

And, `golangci-lint` also denies the git/internal package. Only the
`git/command.go` can use it carefully.

### There is still a `ToTrustedCmdArgs`, will it still allow developers
to make mistakes and pass untrusted arguments?

Generally speaking, no. Because when using `ToTrustedCmdArgs`, the code
will be very complex (see the changes for examples). Then developers and
reviewers can know that something might be unreasonable.

### Why there was a `CmdArgCheck` and why it's removed?

At the moment of #21535, to reduce unnecessary changes, `CmdArgCheck`
was introduced as a hacky patch. Now, almost all code could be written
as `cmd := NewCommand(); cmd.AddXxx(...)`, then there is no need for
`CmdArgCheck` anymore.


### Why many codes for `signArg == ""` is deleted?

Because in the old code, `signArg` could never be empty string, it's
either `-S[key-id]` or `--no-gpg-sign`. So the `signArg == ""` is just
dead code.

---------

Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
2023-02-04 10:30:43 +08:00

504 lines
15 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package files
import (
"bytes"
"context"
"fmt"
"path"
"strings"
"time"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/models/db"
git_model "code.gitea.io/gitea/models/git"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/charset"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
asymkey_service "code.gitea.io/gitea/services/asymkey"
stdcharset "golang.org/x/net/html/charset"
"golang.org/x/text/transform"
)
// IdentityOptions for a person's identity like an author or committer
type IdentityOptions struct {
Name string
Email string
}
// CommitDateOptions store dates for GIT_AUTHOR_DATE and GIT_COMMITTER_DATE
type CommitDateOptions struct {
Author time.Time
Committer time.Time
}
// UpdateRepoFileOptions holds the repository file update options
type UpdateRepoFileOptions struct {
LastCommitID string
OldBranch string
NewBranch string
TreePath string
FromTreePath string
Message string
Content string
SHA string
IsNewFile bool
Author *IdentityOptions
Committer *IdentityOptions
Dates *CommitDateOptions
Signoff bool
}
func detectEncodingAndBOM(entry *git.TreeEntry, repo *repo_model.Repository) (string, bool) {
reader, err := entry.Blob().DataAsync()
if err != nil {
// return default
return "UTF-8", false
}
defer reader.Close()
buf := make([]byte, 1024)
n, err := util.ReadAtMost(reader, buf)
if err != nil {
// return default
return "UTF-8", false
}
buf = buf[:n]
if setting.LFS.StartServer {
pointer, _ := lfs.ReadPointerFromBuffer(buf)
if pointer.IsValid() {
meta, err := git_model.GetLFSMetaObjectByOid(db.DefaultContext, repo.ID, pointer.Oid)
if err != nil && err != git_model.ErrLFSObjectNotExist {
// return default
return "UTF-8", false
}
if meta != nil {
dataRc, err := lfs.ReadMetaObject(pointer)
if err != nil {
// return default
return "UTF-8", false
}
defer dataRc.Close()
buf = make([]byte, 1024)
n, err = util.ReadAtMost(dataRc, buf)
if err != nil {
// return default
return "UTF-8", false
}
buf = buf[:n]
}
}
}
encoding, err := charset.DetectEncoding(buf)
if err != nil {
// just default to utf-8 and no bom
return "UTF-8", false
}
if encoding == "UTF-8" {
return encoding, bytes.Equal(buf[0:3], charset.UTF8BOM)
}
charsetEncoding, _ := stdcharset.Lookup(encoding)
if charsetEncoding == nil {
return "UTF-8", false
}
result, n, err := transform.String(charsetEncoding.NewDecoder(), string(buf))
if err != nil {
// return default
return "UTF-8", false
}
if n > 2 {
return encoding, bytes.Equal([]byte(result)[0:3], charset.UTF8BOM)
}
return encoding, false
}
// CreateOrUpdateRepoFile adds or updates a file in the given repository
func CreateOrUpdateRepoFile(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, opts *UpdateRepoFileOptions) (*structs.FileResponse, error) {
// If no branch name is set, assume default branch
if opts.OldBranch == "" {
opts.OldBranch = repo.DefaultBranch
}
if opts.NewBranch == "" {
opts.NewBranch = opts.OldBranch
}
gitRepo, closer, err := git.RepositoryFromContextOrOpen(ctx, repo.RepoPath())
if err != nil {
return nil, err
}
defer closer.Close()
// oldBranch must exist for this operation
if _, err := gitRepo.GetBranch(opts.OldBranch); err != nil && !repo.IsEmpty {
return nil, err
}
// A NewBranch can be specified for the file to be created/updated in a new branch.
// Check to make sure the branch does not already exist, otherwise we can't proceed.
// If we aren't branching to a new branch, make sure user can commit to the given branch
if opts.NewBranch != opts.OldBranch {
existingBranch, err := gitRepo.GetBranch(opts.NewBranch)
if existingBranch != nil {
return nil, models.ErrBranchAlreadyExists{
BranchName: opts.NewBranch,
}
}
if err != nil && !git.IsErrBranchNotExist(err) {
return nil, err
}
} else if err := VerifyBranchProtection(ctx, repo, doer, opts.OldBranch, opts.TreePath); err != nil {
return nil, err
}
// If FromTreePath is not set, set it to the opts.TreePath
if opts.TreePath != "" && opts.FromTreePath == "" {
opts.FromTreePath = opts.TreePath
}
// Check that the path given in opts.treePath is valid (not a git path)
treePath := CleanUploadFileName(opts.TreePath)
if treePath == "" {
return nil, models.ErrFilenameInvalid{
Path: opts.TreePath,
}
}
// If there is a fromTreePath (we are copying it), also clean it up
fromTreePath := CleanUploadFileName(opts.FromTreePath)
if fromTreePath == "" && opts.FromTreePath != "" {
return nil, models.ErrFilenameInvalid{
Path: opts.FromTreePath,
}
}
message := strings.TrimSpace(opts.Message)
author, committer := GetAuthorAndCommitterUsers(opts.Author, opts.Committer, doer)
t, err := NewTemporaryUploadRepository(ctx, repo)
if err != nil {
log.Error("%v", err)
}
defer t.Close()
hasOldBranch := true
if err := t.Clone(opts.OldBranch); err != nil {
if !git.IsErrBranchNotExist(err) || !repo.IsEmpty {
return nil, err
}
if err := t.Init(); err != nil {
return nil, err
}
hasOldBranch = false
opts.LastCommitID = ""
}
if hasOldBranch {
if err := t.SetDefaultIndex(); err != nil {
return nil, err
}
}
encoding := "UTF-8"
bom := false
executable := false
if hasOldBranch {
// Get the commit of the original branch
commit, err := t.GetBranchCommit(opts.OldBranch)
if err != nil {
return nil, err // Couldn't get a commit for the branch
}
// Assigned LastCommitID in opts if it hasn't been set
if opts.LastCommitID == "" {
opts.LastCommitID = commit.ID.String()
} else {
lastCommitID, err := t.gitRepo.ConvertToSHA1(opts.LastCommitID)
if err != nil {
return nil, fmt.Errorf("ConvertToSHA1: Invalid last commit ID: %w", err)
}
opts.LastCommitID = lastCommitID.String()
}
if !opts.IsNewFile {
fromEntry, err := commit.GetTreeEntryByPath(fromTreePath)
if err != nil {
return nil, err
}
if opts.SHA != "" {
// If a SHA was given and the SHA given doesn't match the SHA of the fromTreePath, throw error
if opts.SHA != fromEntry.ID.String() {
return nil, models.ErrSHADoesNotMatch{
Path: treePath,
GivenSHA: opts.SHA,
CurrentSHA: fromEntry.ID.String(),
}
}
} else if opts.LastCommitID != "" {
// If a lastCommitID was given and it doesn't match the commitID of the head of the branch throw
// an error, but only if we aren't creating a new branch.
if commit.ID.String() != opts.LastCommitID && opts.OldBranch == opts.NewBranch {
if changed, err := commit.FileChangedSinceCommit(treePath, opts.LastCommitID); err != nil {
return nil, err
} else if changed {
return nil, models.ErrCommitIDDoesNotMatch{
GivenCommitID: opts.LastCommitID,
CurrentCommitID: opts.LastCommitID,
}
}
// The file wasn't modified, so we are good to delete it
}
} else {
// When updating a file, a lastCommitID or SHA needs to be given to make sure other commits
// haven't been made. We throw an error if one wasn't provided.
return nil, models.ErrSHAOrCommitIDNotProvided{}
}
encoding, bom = detectEncodingAndBOM(fromEntry, repo)
executable = fromEntry.IsExecutable()
}
// For the path where this file will be created/updated, we need to make
// sure no parts of the path are existing files or links except for the last
// item in the path which is the file name, and that shouldn't exist IF it is
// a new file OR is being moved to a new path.
treePathParts := strings.Split(treePath, "/")
subTreePath := ""
for index, part := range treePathParts {
subTreePath = path.Join(subTreePath, part)
entry, err := commit.GetTreeEntryByPath(subTreePath)
if err != nil {
if git.IsErrNotExist(err) {
// Means there is no item with that name, so we're good
break
}
return nil, err
}
if index < len(treePathParts)-1 {
if !entry.IsDir() {
return nil, models.ErrFilePathInvalid{
Message: fmt.Sprintf("a file exists where youre trying to create a subdirectory [path: %s]", subTreePath),
Path: subTreePath,
Name: part,
Type: git.EntryModeBlob,
}
}
} else if entry.IsLink() {
return nil, models.ErrFilePathInvalid{
Message: fmt.Sprintf("a symbolic link exists where youre trying to create a subdirectory [path: %s]", subTreePath),
Path: subTreePath,
Name: part,
Type: git.EntryModeSymlink,
}
} else if entry.IsDir() {
return nil, models.ErrFilePathInvalid{
Message: fmt.Sprintf("a directory exists where youre trying to create a file [path: %s]", subTreePath),
Path: subTreePath,
Name: part,
Type: git.EntryModeTree,
}
} else if fromTreePath != treePath || opts.IsNewFile {
// The entry shouldn't exist if we are creating new file or moving to a new path
return nil, models.ErrRepoFileAlreadyExists{
Path: treePath,
}
}
}
}
// Get the two paths (might be the same if not moving) from the index if they exist
filesInIndex, err := t.LsFiles(opts.TreePath, opts.FromTreePath)
if err != nil {
return nil, fmt.Errorf("UpdateRepoFile: %w", err)
}
// If is a new file (not updating) then the given path shouldn't exist
if opts.IsNewFile {
for _, file := range filesInIndex {
if file == opts.TreePath {
return nil, models.ErrRepoFileAlreadyExists{
Path: opts.TreePath,
}
}
}
}
// Remove the old path from the tree
if fromTreePath != treePath && len(filesInIndex) > 0 {
for _, file := range filesInIndex {
if file == fromTreePath {
if err := t.RemoveFilesFromIndex(opts.FromTreePath); err != nil {
return nil, err
}
}
}
}
content := opts.Content
if bom {
content = string(charset.UTF8BOM) + content
}
if encoding != "UTF-8" {
charsetEncoding, _ := stdcharset.Lookup(encoding)
if charsetEncoding != nil {
result, _, err := transform.String(charsetEncoding.NewEncoder(), content)
if err != nil {
// Look if we can't encode back in to the original we should just stick with utf-8
log.Error("Error re-encoding %s (%s) as %s - will stay as UTF-8: %v", opts.TreePath, opts.FromTreePath, encoding, err)
result = content
}
content = result
} else {
log.Error("Unknown encoding: %s", encoding)
}
}
// Reset the opts.Content to our adjusted content to ensure that LFS gets the correct content
opts.Content = content
var lfsMetaObject *git_model.LFSMetaObject
if setting.LFS.StartServer && hasOldBranch {
// Check there is no way this can return multiple infos
filename2attribute2info, err := t.gitRepo.CheckAttribute(git.CheckAttributeOpts{
Attributes: []string{"filter"},
Filenames: []string{treePath},
CachedOnly: true,
})
if err != nil {
return nil, err
}
if filename2attribute2info[treePath] != nil && filename2attribute2info[treePath]["filter"] == "lfs" {
// OK so we are supposed to LFS this data!
pointer, err := lfs.GeneratePointer(strings.NewReader(opts.Content))
if err != nil {
return nil, err
}
lfsMetaObject = &git_model.LFSMetaObject{Pointer: pointer, RepositoryID: repo.ID}
content = pointer.StringContent()
}
}
// Add the object to the database
objectHash, err := t.HashObject(strings.NewReader(content))
if err != nil {
return nil, err
}
// Add the object to the index
if executable {
if err := t.AddObjectToIndex("100755", objectHash, treePath); err != nil {
return nil, err
}
} else {
if err := t.AddObjectToIndex("100644", objectHash, treePath); err != nil {
return nil, err
}
}
// Now write the tree
treeHash, err := t.WriteTree()
if err != nil {
return nil, err
}
// Now commit the tree
var commitHash string
if opts.Dates != nil {
commitHash, err = t.CommitTreeWithDate(opts.LastCommitID, author, committer, treeHash, message, opts.Signoff, opts.Dates.Author, opts.Dates.Committer)
} else {
commitHash, err = t.CommitTree(opts.LastCommitID, author, committer, treeHash, message, opts.Signoff)
}
if err != nil {
return nil, err
}
if lfsMetaObject != nil {
// We have an LFS object - create it
lfsMetaObject, err = git_model.NewLFSMetaObject(ctx, lfsMetaObject)
if err != nil {
return nil, err
}
contentStore := lfs.NewContentStore()
exist, err := contentStore.Exists(lfsMetaObject.Pointer)
if err != nil {
return nil, err
}
if !exist {
if err := contentStore.Put(lfsMetaObject.Pointer, strings.NewReader(opts.Content)); err != nil {
if _, err2 := git_model.RemoveLFSMetaObjectByOid(ctx, repo.ID, lfsMetaObject.Oid); err2 != nil {
return nil, fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %w)", lfsMetaObject.Oid, err2, err)
}
return nil, err
}
}
}
// Then push this tree to NewBranch
if err := t.Push(doer, commitHash, opts.NewBranch); err != nil {
log.Error("%T %v", err, err)
return nil, err
}
commit, err := t.GetCommit(commitHash)
if err != nil {
return nil, err
}
file, err := GetFileResponseFromCommit(ctx, repo, commit, opts.NewBranch, treePath)
if err != nil {
return nil, err
}
return file, nil
}
// VerifyBranchProtection verify the branch protection for modifying the given treePath on the given branch
func VerifyBranchProtection(ctx context.Context, repo *repo_model.Repository, doer *user_model.User, branchName, treePath string) error {
protectedBranch, err := git_model.GetFirstMatchProtectedBranchRule(ctx, repo.ID, branchName)
if err != nil {
return err
}
if protectedBranch != nil {
protectedBranch.Repo = repo
isUnprotectedFile := false
glob := protectedBranch.GetUnprotectedFilePatterns()
if len(glob) != 0 {
isUnprotectedFile = protectedBranch.IsUnprotectedFile(glob, treePath)
}
if !protectedBranch.CanUserPush(ctx, doer) && !isUnprotectedFile {
return models.ErrUserCannotCommit{
UserName: doer.LowerName,
}
}
if protectedBranch.RequireSignedCommits {
_, _, _, err := asymkey_service.SignCRUDAction(ctx, repo.RepoPath(), doer, repo.RepoPath(), branchName)
if err != nil {
if !asymkey_service.IsErrWontSign(err) {
return err
}
return models.ErrUserCannotCommit{
UserName: doer.LowerName,
}
}
}
patterns := protectedBranch.GetProtectedFilePatterns()
for _, pat := range patterns {
if pat.Match(strings.ToLower(treePath)) {
return models.ErrFilePathProtected{
Path: treePath,
}
}
}
}
return nil
}