From 2fbf9b23dc9a3e0403284d04e849d6bc9d7f4896 Mon Sep 17 00:00:00 2001 From: Steve Fan <29133953+stevefan1999-personal@users.noreply.github.com> Date: Fri, 11 Nov 2022 15:33:08 +0000 Subject: [PATCH 1/8] storage, minio: add options to disable signature and multipart --- modules/setting/storage.go | 2 ++ modules/storage/minio.go | 16 +++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/modules/setting/storage.go b/modules/setting/storage.go index 075152db59..ca26c3036c 100644 --- a/modules/setting/storage.go +++ b/modules/setting/storage.go @@ -42,6 +42,8 @@ func getStorage(name, typ string, targetSec *ini.Section) Storage { sec.Key("MINIO_BUCKET").MustString("gitea") sec.Key("MINIO_LOCATION").MustString("us-east-1") sec.Key("MINIO_USE_SSL").MustBool(false) + sec.Key("MINIO_DISABLE_SIGNATURE").MustBool(false) + sec.Key("MINIO_DISABLE_MULTIPART").MustBool(false) if targetSec == nil { targetSec, _ = Cfg.NewSection(name) diff --git a/modules/storage/minio.go b/modules/storage/minio.go index f7b42d674c..f40f1e7d6d 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -43,13 +43,15 @@ const MinioStorageType Type = "minio" // MinioStorageConfig represents the configuration for a minio storage type MinioStorageConfig struct { - Endpoint string `ini:"MINIO_ENDPOINT"` - AccessKeyID string `ini:"MINIO_ACCESS_KEY_ID"` - SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY"` - Bucket string `ini:"MINIO_BUCKET"` - Location string `ini:"MINIO_LOCATION"` - BasePath string `ini:"MINIO_BASE_PATH"` - UseSSL bool `ini:"MINIO_USE_SSL"` + Endpoint string `ini:"MINIO_ENDPOINT"` + AccessKeyID string `ini:"MINIO_ACCESS_KEY_ID"` + SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY"` + Bucket string `ini:"MINIO_BUCKET"` + Location string `ini:"MINIO_LOCATION"` + BasePath string `ini:"MINIO_BASE_PATH"` + UseSSL bool `ini:"MINIO_USE_SSL"` + DisableSignature bool `ini:"MINIO_DISABLE_SIGNATURE"` + DisableMultipart bool `ini:"MINIO_DISABLE_MULTIPART"` } // MinioStorage returns a minio bucket storage From d61714c8abb8868080f26fdec4db93c22f2114c7 Mon Sep 17 00:00:00 2001 From: Steve Fan <29133953+stevefan1999-personal@users.noreply.github.com> Date: Fri, 11 Nov 2022 15:34:57 +0000 Subject: [PATCH 2/8] storage/minio: save pointer to conifg --- modules/storage/minio.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/storage/minio.go b/modules/storage/minio.go index f40f1e7d6d..4cc94d56ca 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -60,6 +60,7 @@ type MinioStorage struct { client *minio.Client bucket string basePath string + config *MinioStorageConfig } func convertMinioErr(err error) error { @@ -115,6 +116,7 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error client: minioClient, bucket: config.Bucket, basePath: config.BasePath, + config: &config, }, nil } From c71fa9b8db08e26366c18b2e630124b7ce0f45db Mon Sep 17 00:00:00 2001 From: Steve Fan <29133953+stevefan1999-personal@users.noreply.github.com> Date: Fri, 11 Nov 2022 15:39:51 +0000 Subject: [PATCH 3/8] storage/minio: put the disable signature and multipart in consideration for PutObject --- modules/storage/minio.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 4cc94d56ca..c2a63fd589 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -136,13 +136,18 @@ func (m *MinioStorage) Open(path string) (Object, error) { // Save save a file to minio func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) { + disableSignature, disableMultipart := false, false + if m.config != nil { + disableSignature, disableMultipart = m.config.DisableSignature, m.config.DisableMultipart + } + uploadInfo, err := m.client.PutObject( m.ctx, m.bucket, m.buildMinioPath(path), r, size, - minio.PutObjectOptions{ContentType: "application/octet-stream"}, + minio.PutObjectOptions{ContentType: "application/octet-stream", DisableContentSha256: disableSignature, DisableMultipart: disableMultipart}, ) if err != nil { return 0, convertMinioErr(err) From 866f567a3d27ac38bf7f1fcc3db84136e1511287 Mon Sep 17 00:00:00 2001 From: Steve Fan <29133953+stevefan1999-personal@users.noreply.github.com> Date: Fri, 11 Nov 2022 16:04:34 +0000 Subject: [PATCH 4/8] storage/minio: add workaround for unspecified size in Save when multipart is disabled --- modules/storage/minio.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/modules/storage/minio.go b/modules/storage/minio.go index c2a63fd589..f1a6a62141 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -5,6 +5,7 @@ package storage import ( + "bytes" "context" "io" "net/url" @@ -141,6 +142,22 @@ func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) disableSignature, disableMultipart = m.config.DisableSignature, m.config.DisableMultipart } + if disableMultipart && size < 0 { + // Attempts to read everything from the source into memory. This can take a big toll on memory, and it can become a potential DoS source + // but since we have disabled multipart upload this mean we can't really stream write anymore... + // well, unless we have a better way to estimate the stream size, this would be a workaround + + buf := &bytes.Buffer{} + if n, err := io.Copy(buf, r); err != nil { + // I guess this would likely be EOF or OOM...? + return -1, err + } else { + // Since we read all the data from the source, it might not be usable again, + // so we should swap the reader location to our memory buffer + r, size = buf, n + } + } + uploadInfo, err := m.client.PutObject( m.ctx, m.bucket, From e3e96e7a66ed7b35f5d7516625e43d26df9efc08 Mon Sep 17 00:00:00 2001 From: Steve Fan <29133953+stevefan1999-personal@users.noreply.github.com> Date: Fri, 11 Nov 2022 16:11:08 +0000 Subject: [PATCH 5/8] storage/minio: adapt to linter advice --- modules/storage/minio.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/modules/storage/minio.go b/modules/storage/minio.go index f1a6a62141..926c089508 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -148,14 +148,16 @@ func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) // well, unless we have a better way to estimate the stream size, this would be a workaround buf := &bytes.Buffer{} - if n, err := io.Copy(buf, r); err != nil { + n, err := io.Copy(buf, r) + + if err != nil { // I guess this would likely be EOF or OOM...? return -1, err - } else { - // Since we read all the data from the source, it might not be usable again, - // so we should swap the reader location to our memory buffer - r, size = buf, n } + + // Since we read all the data from the source, it might not be usable again, + // so we should swap the reader location to our memory buffer + r, size = buf, n } uploadInfo, err := m.client.PutObject( From f88da9189d2e53c4bd475541bfe28ecd293c0e67 Mon Sep 17 00:00:00 2001 From: Steve Fan <29133953+stevefan1999-personal@users.noreply.github.com> Date: Fri, 11 Nov 2022 16:38:47 +0000 Subject: [PATCH 6/8] storage/minio: fix extra line reported from linter --- modules/storage/minio.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 926c089508..5eeec362c7 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -149,7 +149,6 @@ func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) buf := &bytes.Buffer{} n, err := io.Copy(buf, r) - if err != nil { // I guess this would likely be EOF or OOM...? return -1, err From d84b1ee9456a7917d6f0b15cd3fbcb0b8f0bd380 Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Tue, 23 May 2023 16:34:23 -0400 Subject: [PATCH 7/8] Update modules/storage/minio.go --- modules/storage/minio.go | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 6e69e05a3f..8d999b2889 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -46,7 +46,6 @@ const MinioStorageType Type = "minio" // MinioStorageConfig represents the configuration for a minio storage type MinioStorageConfig struct { - Endpoint string `ini:"MINIO_ENDPOINT"` AccessKeyID string `ini:"MINIO_ACCESS_KEY_ID"` SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY"` From 635e997e50e1dcaba7b64b801ba5f4a68007c1b2 Mon Sep 17 00:00:00 2001 From: Steve Fan <29133953+stevefan1999-personal@users.noreply.github.com> Date: Tue, 22 Aug 2023 14:24:26 +0800 Subject: [PATCH 8/8] Update minio.go --- modules/storage/minio.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/storage/minio.go b/modules/storage/minio.go index 360663574c..909ea51a18 100644 --- a/modules/storage/minio.go +++ b/modules/storage/minio.go @@ -50,7 +50,6 @@ type MinioStorage struct { client *minio.Client bucket string basePath string - config *MinioStorageConfig } func convertMinioErr(err error) error { @@ -112,7 +111,6 @@ func NewMinioStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage, client: minioClient, bucket: config.Bucket, basePath: config.BasePath, - config: &config, }, nil } @@ -137,8 +135,8 @@ func (m *MinioStorage) Open(path string) (Object, error) { // Save saves a file to minio func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) { disableSignature, disableMultipart := false, false - if m.config != nil { - disableSignature, disableMultipart = m.config.DisableSignature, m.config.DisableMultipart + if m.cfg != nil { + disableSignature, disableMultipart = m.cfg.DisableSignature, m.cfg.DisableMultipart } if disableMultipart && size < 0 { @@ -146,6 +144,8 @@ func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) // but since we have disabled multipart upload this mean we can't really stream write anymore... // well, unless we have a better way to estimate the stream size, this would be a workaround + // another alternative: we can use mmap instead, but this would be very dangerous as it is platform-specific + buf := &bytes.Buffer{} n, err := io.Copy(buf, r) if err != nil { @@ -171,8 +171,8 @@ func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) // * https://www.backblaze.com/b2/docs/s3_compatible_api.html // do not support "x-amz-checksum-algorithm" header, so use legacy MD5 checksum SendContentMd5: m.cfg.ChecksumAlgorithm == "md5", - DisableContentSha256: m.cfg.DisableSignature, - DisableMultipart: m.cfg.DisableMultipart + DisableContentSha256: disableSignature, + DisableMultipart: disableMultipart }, ) if err != nil {