From bb720c87c191efe36a328d95a918f75df51d4976 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Guillot?= Date: Sun, 2 Jun 2019 07:13:35 -0700 Subject: [PATCH] Make HTTP Client timeout and max body size configurable --- config/config_test.go | 66 +++++++++++++++++++++++++++++++++++++++++++ config/options.go | 52 +++++++++++++++++++++------------- config/parser.go | 7 +++-- http/client/client.go | 15 +++------- miniflux.1 | 10 +++++++ 5 files changed, 117 insertions(+), 33 deletions(-) diff --git a/config/config_test.go b/config/config_test.go index 0e691fc0..6b6cdf2e 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -984,3 +984,69 @@ func TestHTTPSOn(t *testing.T) { t.Fatalf(`Unexpected HTTPS value, got "%v"`, opts.HTTPS) } } + +func TestHTTPClientTimeout(t *testing.T) { + os.Clearenv() + os.Setenv("HTTP_CLIENT_TIMEOUT", "42") + + opts, err := parse() + if err != nil { + t.Fatalf(`Parsing failure: %q`, err) + } + + expected := 42 + result := opts.HTTPClientTimeout() + + if result != expected { + t.Fatalf(`Unexpected HTTP_CLIENT_TIMEOUT value, got %d instead of %d`, result, expected) + } +} + +func TestDefaultHTTPClientTimeoutValue(t *testing.T) { + os.Clearenv() + + opts, err := parse() + if err != nil { + t.Fatalf(`Parsing failure: %q`, err) + } + + expected := defaultHTTPClientTimeout + result := opts.HTTPClientTimeout() + + if result != expected { + t.Fatalf(`Unexpected HTTP_CLIENT_TIMEOUT value, got %d instead of %d`, result, expected) + } +} + +func TestHTTPClientMaxBodySize(t *testing.T) { + os.Clearenv() + os.Setenv("HTTP_CLIENT_MAX_BODY_SIZE", "42") + + opts, err := parse() + if err != nil { + t.Fatalf(`Parsing failure: %q`, err) + } + + expected := int64(42 * 1024 * 1024) + result := opts.HTTPClientMaxBodySize() + + if result != expected { + t.Fatalf(`Unexpected HTTP_CLIENT_MAX_BODY_SIZE value, got %d instead of %d`, result, expected) + } +} + +func TestDefaultHTTPClientMaxBodySizeValue(t *testing.T) { + os.Clearenv() + + opts, err := parse() + if err != nil { + t.Fatalf(`Parsing failure: %q`, err) + } + + expected := int64(defaultHTTPClientMaxBodySize * 1024 * 1024) + result := opts.HTTPClientMaxBodySize() + + if result != expected { + t.Fatalf(`Unexpected HTTP_CLIENT_MAX_BODY_SIZE value, got %d instead of %d`, result, expected) + } +} diff --git a/config/options.go b/config/options.go index ad8d2c7c..32105e9b 100644 --- a/config/options.go +++ b/config/options.go @@ -5,25 +5,27 @@ package config // import "miniflux.app/config" const ( - defaultBaseURL = "http://localhost" - defaultWorkerPoolSize = 5 - defaultPollingFrequency = 60 - defaultBatchSize = 10 - defaultDatabaseURL = "user=postgres password=postgres dbname=miniflux2 sslmode=disable" - defaultDatabaseMaxConns = 20 - defaultDatabaseMinConns = 1 - defaultArchiveReadDays = 60 - defaultListenAddr = "127.0.0.1:8080" - defaultCertFile = "" - defaultKeyFile = "" - defaultCertDomain = "" - defaultCertCache = "/tmp/cert_cache" - defaultCleanupFrequency = 24 - defaultProxyImages = "http-only" - defaultOAuth2ClientID = "" - defaultOAuth2ClientSecret = "" - defaultOAuth2RedirectURL = "" - defaultOAuth2Provider = "" + defaultBaseURL = "http://localhost" + defaultWorkerPoolSize = 5 + defaultPollingFrequency = 60 + defaultBatchSize = 10 + defaultDatabaseURL = "user=postgres password=postgres dbname=miniflux2 sslmode=disable" + defaultDatabaseMaxConns = 20 + defaultDatabaseMinConns = 1 + defaultArchiveReadDays = 60 + defaultListenAddr = "127.0.0.1:8080" + defaultCertFile = "" + defaultKeyFile = "" + defaultCertDomain = "" + defaultCertCache = "/tmp/cert_cache" + defaultCleanupFrequency = 24 + defaultProxyImages = "http-only" + defaultOAuth2ClientID = "" + defaultOAuth2ClientSecret = "" + defaultOAuth2RedirectURL = "" + defaultOAuth2Provider = "" + defaultHTTPClientTimeout = 20 + defaultHTTPClientMaxBodySize = 15 ) // Options contains configuration options. @@ -58,6 +60,8 @@ type Options struct { oauth2RedirectURL string oauth2Provider string pocketConsumerKey string + httpClientTimeout int + httpClientMaxBodySize int64 } // HasDebugMode returns true if debug mode is enabled. @@ -212,3 +216,13 @@ func (o *Options) PocketConsumerKey(defaultValue string) string { } return defaultValue } + +// HTTPClientTimeout returns the time limit in seconds before the HTTP client cancel the request. +func (o *Options) HTTPClientTimeout() int { + return o.httpClientTimeout +} + +// HTTPClientMaxBodySize returns the number of bytes allowed for the HTTP client to transfer. +func (o *Options) HTTPClientMaxBodySize() int64 { + return o.httpClientMaxBodySize +} diff --git a/config/parser.go b/config/parser.go index 996d0a80..b2ed2e76 100644 --- a/config/parser.go +++ b/config/parser.go @@ -45,6 +45,8 @@ func parse() (opts *Options, err error) { opts.batchSize = getIntValue("BATCH_SIZE", defaultBatchSize) opts.archiveReadDays = getIntValue("ARCHIVE_READ_DAYS", defaultArchiveReadDays) opts.proxyImages = getStringValue("PROXY_IMAGES", defaultProxyImages) + opts.createAdmin = getBooleanValue("CREATE_ADMIN") + opts.pocketConsumerKey = getStringValue("POCKET_CONSUMER_KEY", "") opts.oauth2UserCreationAllowed = getBooleanValue("OAUTH2_USER_CREATION") opts.oauth2ClientID = getStringValue("OAUTH2_CLIENT_ID", defaultOAuth2ClientID) @@ -52,9 +54,8 @@ func parse() (opts *Options, err error) { opts.oauth2RedirectURL = getStringValue("OAUTH2_REDIRECT_URL", defaultOAuth2RedirectURL) opts.oauth2Provider = getStringValue("OAUTH2_PROVIDER", defaultOAuth2Provider) - opts.pocketConsumerKey = getStringValue("POCKET_CONSUMER_KEY", "") - - opts.createAdmin = getBooleanValue("CREATE_ADMIN") + opts.httpClientTimeout = getIntValue("HTTP_CLIENT_TIMEOUT", defaultHTTPClientTimeout) + opts.httpClientMaxBodySize = int64(getIntValue("HTTP_CLIENT_MAX_BODY_SIZE", defaultHTTPClientMaxBodySize) * 1024 * 1024) return opts, nil } diff --git a/http/client/client.go b/http/client/client.go index 2dce15c3..175fe9ff 100644 --- a/http/client/client.go +++ b/http/client/client.go @@ -18,20 +18,13 @@ import ( "strings" "time" + "miniflux.app/config" "miniflux.app/errors" "miniflux.app/logger" "miniflux.app/timer" "miniflux.app/version" ) -const ( - // 20 seconds max. - requestTimeout = 20 - - // 15MB max. - maxBodySize = 1024 * 1024 * 15 -) - var ( // DefaultUserAgent sets the User-Agent header used for any requests by miniflux. DefaultUserAgent = "Mozilla/5.0 (compatible; Miniflux/" + version.Version + "; +https://miniflux.app)" @@ -144,7 +137,7 @@ func (c *Client) executeRequest(request *http.Request) (*Response, error) { case net.Error: nerr := uerr.Err.(net.Error) if nerr.Timeout() { - err = errors.NewLocalizedError(errRequestTimeout, requestTimeout) + err = errors.NewLocalizedError(errRequestTimeout, config.Opts.HTTPClientTimeout()) } else if nerr.Temporary() { err = errors.NewLocalizedError(errTemporaryNetworkOperation, nerr) } @@ -154,7 +147,7 @@ func (c *Client) executeRequest(request *http.Request) (*Response, error) { return nil, err } - if resp.ContentLength > maxBodySize { + if resp.ContentLength > config.Opts.HTTPClientMaxBodySize() { return nil, fmt.Errorf("client: response too large (%d bytes)", resp.ContentLength) } @@ -212,7 +205,7 @@ func (c *Client) buildRequest(method string, body io.Reader) (*http.Request, err } func (c *Client) buildClient() http.Client { - client := http.Client{Timeout: time.Duration(requestTimeout * time.Second)} + client := http.Client{Timeout: time.Duration(config.Opts.HTTPClientTimeout()) * time.Second} if c.Insecure { client.Transport = &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, diff --git a/miniflux.1 b/miniflux.1 index 503f2cfa..911732df 100644 --- a/miniflux.1 +++ b/miniflux.1 @@ -169,6 +169,16 @@ Pocket consumer API key for all users\&. Avoids mixed content warnings for external images: http-only, all, or none\&. .br Default is http-only\&. +.TP +.B HTTP_CLIENT_TIMEOUT +Time limit in seconds before the HTTP client cancel the request\&. +.br +Default is 20 seconds\&. +.TP +.B HTTP_CLIENT_MAX_BODY_SIZE +Maximum body size for HTTP requests in Mebibyte (MiB)\&. +.br +Default is 20 MiB\&. .SH AUTHORS .sp