// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. // SPDX-License-Identifier: Apache-2.0 package client // import "miniflux.app/v2/internal/http/client" import ( "bytes" "crypto/tls" "crypto/x509" "fmt" "io" "log/slog" "net" "net/http" "net/url" "time" "miniflux.app/v2/internal/config" "miniflux.app/v2/internal/errors" ) const ( defaultHTTPClientTimeout = 20 defaultHTTPClientMaxBodySize = 15 * 1024 * 1024 ) var ( errInvalidCertificate = "Invalid SSL certificate (original error: %q)" errNetworkOperation = "This website is unreachable (original error: %q)" errRequestTimeout = "Website unreachable, the request timed out after %d seconds" ) // Client builds and executes HTTP requests. type Client struct { inputURL string requestEtagHeader string requestLastModifiedHeader string requestAuthorizationHeader string requestUsername string requestPassword string requestUserAgent string requestCookie string customHeaders map[string]string useProxy bool doNotFollowRedirects bool ClientTimeout int ClientMaxBodySize int64 ClientProxyURL string AllowSelfSignedCertificates bool } // New initializes a new HTTP client. func New(url string) *Client { return &Client{ inputURL: url, ClientTimeout: defaultHTTPClientTimeout, ClientMaxBodySize: defaultHTTPClientMaxBodySize, } } // NewClientWithConfig initializes a new HTTP client with application config options. func NewClientWithConfig(url string, opts *config.Options) *Client { return &Client{ inputURL: url, requestUserAgent: opts.HTTPClientUserAgent(), ClientTimeout: opts.HTTPClientTimeout(), ClientMaxBodySize: opts.HTTPClientMaxBodySize(), ClientProxyURL: opts.HTTPClientProxy(), } } // WithCredentials defines the username/password for HTTP Basic authentication. func (c *Client) WithCredentials(username, password string) *Client { if username != "" && password != "" { c.requestUsername = username c.requestPassword = password } return c } // WithCustomHeaders defines custom HTTP headers. func (c *Client) WithCustomHeaders(customHeaders map[string]string) *Client { c.customHeaders = customHeaders return c } // WithCacheHeaders defines caching headers. func (c *Client) WithCacheHeaders(etagHeader, lastModifiedHeader string) *Client { c.requestEtagHeader = etagHeader c.requestLastModifiedHeader = lastModifiedHeader return c } // WithProxy enables proxy for the current HTTP request. func (c *Client) WithProxy() *Client { c.useProxy = true return c } // WithoutRedirects disables HTTP redirects. func (c *Client) WithoutRedirects() *Client { c.doNotFollowRedirects = true return c } // WithUserAgent defines the User-Agent header to use for HTTP requests. func (c *Client) WithUserAgent(userAgent string) *Client { if userAgent != "" { c.requestUserAgent = userAgent } return c } // WithCookie defines the Cookies to use for HTTP requests. func (c *Client) WithCookie(cookie string) *Client { if cookie != "" { c.requestCookie = cookie } return c } // Get performs a GET HTTP request. func (c *Client) Get() (*Response, error) { request, err := c.buildRequest(http.MethodGet, nil) if err != nil { return nil, err } return c.executeRequest(request) } func (c *Client) executeRequest(request *http.Request) (*Response, error) { startTime := time.Now() slog.Debug("Executing outgoing HTTP request", slog.Group("request", slog.String("method", request.Method), slog.String("url", request.URL.String()), slog.String("user_agent", request.UserAgent()), slog.Bool("is_authenticated", c.requestAuthorizationHeader != "" || (c.requestUsername != "" && c.requestPassword != "")), slog.Bool("has_cookie", c.requestCookie != ""), slog.Bool("with_redirects", !c.doNotFollowRedirects), slog.Bool("with_proxy", c.useProxy), slog.String("proxy_url", c.ClientProxyURL), slog.Bool("with_caching_headers", c.requestEtagHeader != "" || c.requestLastModifiedHeader != ""), ), ) client := c.buildClient() resp, err := client.Do(request) if resp != nil { defer resp.Body.Close() } if err != nil { if uerr, ok := err.(*url.Error); ok { switch uerr.Err.(type) { case x509.CertificateInvalidError, x509.HostnameError: err = errors.NewLocalizedError(errInvalidCertificate, uerr.Err) case *net.OpError: err = errors.NewLocalizedError(errNetworkOperation, uerr.Err) case net.Error: nerr := uerr.Err.(net.Error) if nerr.Timeout() { err = errors.NewLocalizedError(errRequestTimeout, c.ClientTimeout) } } } return nil, err } if resp.ContentLength > c.ClientMaxBodySize { return nil, fmt.Errorf("client: response too large (%d bytes)", resp.ContentLength) } buf, err := io.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("client: error while reading body %v", err) } response := &Response{ Body: bytes.NewReader(buf), StatusCode: resp.StatusCode, EffectiveURL: resp.Request.URL.String(), LastModified: resp.Header.Get("Last-Modified"), ETag: resp.Header.Get("ETag"), Expires: resp.Header.Get("Expires"), ContentType: resp.Header.Get("Content-Type"), ContentLength: resp.ContentLength, } slog.Debug("Completed outgoing HTTP request", slog.Duration("duration", time.Since(startTime)), slog.Group("request", slog.String("method", request.Method), slog.String("url", request.URL.String()), slog.String("user_agent", request.UserAgent()), slog.Bool("is_authenticated", c.requestAuthorizationHeader != "" || (c.requestUsername != "" && c.requestPassword != "")), slog.Bool("has_cookie", c.requestCookie != ""), slog.Bool("with_redirects", !c.doNotFollowRedirects), slog.Bool("with_proxy", c.useProxy), slog.String("proxy_url", c.ClientProxyURL), slog.Bool("with_caching_headers", c.requestEtagHeader != "" || c.requestLastModifiedHeader != ""), ), slog.Group("response", slog.Int("status_code", response.StatusCode), slog.String("effective_url", response.EffectiveURL), slog.String("content_type", response.ContentType), slog.Int64("content_length", response.ContentLength), slog.String("last_modified", response.LastModified), slog.String("etag", response.ETag), slog.String("expires", response.Expires), ), ) // Ignore caching headers for feeds that do not want any cache. if resp.Header.Get("Expires") == "0" { response.ETag = "" response.LastModified = "" } return response, err } func (c *Client) buildRequest(method string, body io.Reader) (*http.Request, error) { request, err := http.NewRequest(method, c.inputURL, body) if err != nil { return nil, err } request.Header = c.buildHeaders() if c.requestUsername != "" && c.requestPassword != "" { request.SetBasicAuth(c.requestUsername, c.requestPassword) } return request, nil } func (c *Client) buildClient() http.Client { client := http.Client{ Timeout: time.Duration(c.ClientTimeout) * time.Second, } transport := &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ // Default is 30s. Timeout: 10 * time.Second, // Default is 30s. KeepAlive: 15 * time.Second, }).DialContext, // Default is 100. MaxIdleConns: 50, // Default is 90s. IdleConnTimeout: 10 * time.Second, } if c.AllowSelfSignedCertificates { transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} } if c.doNotFollowRedirects { client.CheckRedirect = func(req *http.Request, via []*http.Request) error { return http.ErrUseLastResponse } } if c.useProxy && c.ClientProxyURL != "" { proxyURL, err := url.Parse(c.ClientProxyURL) if err != nil { slog.Error("Unable to parse proxy URL", slog.String("proxy_url", c.ClientProxyURL), slog.Any("error", err), ) } else { transport.Proxy = http.ProxyURL(proxyURL) } } client.Transport = transport return client } func (c *Client) buildHeaders() http.Header { headers := make(http.Header) headers.Add("Accept", "*/*") if c.requestUserAgent != "" { headers.Add("User-Agent", c.requestUserAgent) } if c.requestEtagHeader != "" { headers.Add("If-None-Match", c.requestEtagHeader) } if c.requestLastModifiedHeader != "" { headers.Add("If-Modified-Since", c.requestLastModifiedHeader) } if c.requestAuthorizationHeader != "" { headers.Add("Authorization", c.requestAuthorizationHeader) } if c.requestCookie != "" { headers.Add("Cookie", c.requestCookie) } for key, value := range c.customHeaders { headers.Add(key, value) } headers.Add("Connection", "close") return headers }