From 2c2700a31d7349f67016a3786125597f9ee38c56 Mon Sep 17 00:00:00 2001 From: Romain de Laage Date: Sat, 25 Feb 2023 09:36:19 +0100 Subject: [PATCH] Proxy support for several media types closes #615 closes #635 --- api/entry.go | 15 +- config/config_test.go | 141 +++++++++++++++++- config/options.go | 55 +++++-- config/parser.go | 17 ++- fever/handler.go | 2 +- googlereader/handler.go | 13 +- http/response/builder.go | 7 +- http/response/html/html.go | 13 ++ http/response/html/html_test.go | 29 ++++ miniflux.1 | 23 ++- proxy/image_proxy.go | 84 ----------- proxy/media_proxy.go | 123 +++++++++++++++ ...mage_proxy_test.go => media_proxy_test.go} | 132 +++++++++------- proxy/proxy.go | 4 +- service/httpd/httpd.go | 6 +- template/functions.go | 14 +- template/templates/views/entry.html | 14 +- ui/entry_scraper.go | 2 +- ui/proxy.go | 38 ++++- ui/ui.go | 2 +- 20 files changed, 534 insertions(+), 200 deletions(-) delete mode 100644 proxy/image_proxy.go create mode 100644 proxy/media_proxy.go rename proxy/{image_proxy_test.go => media_proxy_test.go} (57%) diff --git a/api/entry.go b/api/entry.go index e8d6c908..e64e867f 100644 --- a/api/entry.go +++ b/api/entry.go @@ -35,12 +35,17 @@ func (h *handler) getEntryFromBuilder(w http.ResponseWriter, r *http.Request, b return } - entry.Content = proxy.AbsoluteImageProxyRewriter(h.router, r.Host, entry.Content) - proxyImage := config.Opts.ProxyImages() + entry.Content = proxy.AbsoluteProxyRewriter(h.router, r.Host, entry.Content) + proxyOption := config.Opts.ProxyOption() for i := range entry.Enclosures { - if strings.HasPrefix(entry.Enclosures[i].MimeType, "image/") && (proxyImage == "all" || proxyImage != "none" && !url.IsHTTPS(entry.Enclosures[i].URL)) { - entry.Enclosures[i].URL = proxy.AbsoluteProxifyURL(h.router, r.Host, entry.Enclosures[i].URL) + if proxyOption == "all" || proxyOption != "none" && !url.IsHTTPS(entry.Enclosures[i].URL) { + for _, mediaType := range config.Opts.ProxyMediaTypes() { + if strings.HasPrefix(entry.Enclosures[i].MimeType, mediaType+"/") { + entry.Enclosures[i].URL = proxy.AbsoluteProxifyURL(h.router, r.Host, entry.Enclosures[i].URL) + break + } + } } } @@ -158,7 +163,7 @@ func (h *handler) findEntries(w http.ResponseWriter, r *http.Request, feedID int } for i := range entries { - entries[i].Content = proxy.AbsoluteImageProxyRewriter(h.router, r.Host, entries[i].Content) + entries[i].Content = proxy.AbsoluteProxyRewriter(h.router, r.Host, entries[i].Content) } json.OK(w, r, &entriesResponse{Total: count, Entries: entries}) diff --git a/config/config_test.go b/config/config_test.go index 188e2b7b..1725caa5 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1163,9 +1163,9 @@ func TestPocketConsumerKeyFromUserPrefs(t *testing.T) { } } -func TestProxyImages(t *testing.T) { +func TestProxyOption(t *testing.T) { os.Clearenv() - os.Setenv("PROXY_IMAGES", "all") + os.Setenv("PROXY_OPTION", "all") parser := NewParser() opts, err := parser.ParseEnvironmentVariables() @@ -1174,14 +1174,14 @@ func TestProxyImages(t *testing.T) { } expected := "all" - result := opts.ProxyImages() + result := opts.ProxyOption() if result != expected { - t.Fatalf(`Unexpected PROXY_IMAGES value, got %q instead of %q`, result, expected) + t.Fatalf(`Unexpected PROXY_OPTION value, got %q instead of %q`, result, expected) } } -func TestDefaultProxyImagesValue(t *testing.T) { +func TestDefaultProxyOptionValue(t *testing.T) { os.Clearenv() parser := NewParser() @@ -1190,11 +1190,101 @@ func TestDefaultProxyImagesValue(t *testing.T) { t.Fatalf(`Parsing failure: %v`, err) } - expected := defaultProxyImages - result := opts.ProxyImages() + expected := defaultProxyOption + result := opts.ProxyOption() if result != expected { - t.Fatalf(`Unexpected PROXY_IMAGES value, got %q instead of %q`, result, expected) + t.Fatalf(`Unexpected PROXY_OPTION value, got %q instead of %q`, result, expected) + } +} + +func TestProxyMediaTypes(t *testing.T) { + os.Clearenv() + os.Setenv("PROXY_MEDIA_TYPES", "image,audio") + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := []string{"audio", "image"} + + if len(expected) != len(opts.ProxyMediaTypes()) { + t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected) + } + + resultMap := make(map[string]bool) + for _, mediaType := range opts.ProxyMediaTypes() { + resultMap[mediaType] = true + } + + for _, mediaType := range expected { + if !resultMap[mediaType] { + t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected) + } + } +} + +func TestDefaultProxyMediaTypes(t *testing.T) { + os.Clearenv() + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := []string{"image"} + + if len(expected) != len(opts.ProxyMediaTypes()) { + t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected) + } + + resultMap := make(map[string]bool) + for _, mediaType := range opts.ProxyMediaTypes() { + resultMap[mediaType] = true + } + + for _, mediaType := range expected { + if !resultMap[mediaType] { + t.Fatalf(`Unexpected PROXY_MEDIA_TYPES value, got %v instead of %v`, opts.ProxyMediaTypes(), expected) + } + } +} + +func TestProxyHTTPClientTimeout(t *testing.T) { + os.Clearenv() + os.Setenv("PROXY_HTTP_CLIENT_TIMEOUT", "24") + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := 24 + result := opts.ProxyHTTPClientTimeout() + + if result != expected { + t.Fatalf(`Unexpected PROXY_HTTP_CLIENT_TIMEOUT value, got %d instead of %d`, result, expected) + } +} + +func TestDefaultProxyHTTPClientTimeoutValue(t *testing.T) { + os.Clearenv() + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := defaultProxyHTTPClientTimeout + result := opts.ProxyHTTPClientTimeout() + + if result != expected { + t.Fatalf(`Unexpected PROXY_HTTP_CLIENT_TIMEOUT value, got %d instead of %d`, result, expected) } } @@ -1297,6 +1387,41 @@ func TestDefaultHTTPClientMaxBodySizeValue(t *testing.T) { } } +func TestHTTPServerTimeout(t *testing.T) { + os.Clearenv() + os.Setenv("HTTP_SERVER_TIMEOUT", "342") + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := 342 + result := opts.HTTPServerTimeout() + + if result != expected { + t.Fatalf(`Unexpected HTTP_SERVER_TIMEOUT value, got %d instead of %d`, result, expected) + } +} + +func TestDefaultHTTPServerTimeoutValue(t *testing.T) { + os.Clearenv() + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := defaultHTTPServerTimeout + result := opts.HTTPServerTimeout() + + if result != expected { + t.Fatalf(`Unexpected HTTP_SERVER_TIMEOUT value, got %d instead of %d`, result, expected) + } +} + func TestParseConfigFile(t *testing.T) { content := []byte(` # This is a comment diff --git a/config/options.go b/config/options.go index 44af9a3e..dea8c726 100644 --- a/config/options.go +++ b/config/options.go @@ -46,8 +46,10 @@ const ( defaultCleanupArchiveUnreadDays = 180 defaultCleanupArchiveBatchSize = 10000 defaultCleanupRemoveSessionsDays = 30 - defaultProxyImages = "http-only" - defaultProxyImageUrl = "" + defaultProxyHTTPClientTimeout = 120 + defaultProxyOption = "http-only" + defaultProxyMediaTypes = "image" + defaultProxyUrl = "" defaultFetchYouTubeWatchTime = false defaultCreateAdmin = false defaultAdminUsername = "" @@ -62,6 +64,7 @@ const ( defaultHTTPClientTimeout = 20 defaultHTTPClientMaxBodySize = 15 defaultHTTPClientProxy = "" + defaultHTTPServerTimeout = 300 defaultAuthProxyHeader = "" defaultAuthProxyUserCreation = false defaultMaintenanceMode = false @@ -117,8 +120,10 @@ type Options struct { createAdmin bool adminUsername string adminPassword string - proxyImages string - proxyImageUrl string + proxyHTTPClientTimeout int + proxyOption string + proxyMediaTypes []string + proxyUrl string fetchYouTubeWatchTime bool oauth2UserCreationAllowed bool oauth2ClientID string @@ -131,6 +136,7 @@ type Options struct { httpClientMaxBodySize int64 httpClientProxy string httpClientUserAgent string + httpServerTimeout int authProxyHeader string authProxyUserCreation bool maintenanceMode bool @@ -181,8 +187,10 @@ func NewOptions() *Options { pollingParsingErrorLimit: defaultPollingParsingErrorLimit, workerPoolSize: defaultWorkerPoolSize, createAdmin: defaultCreateAdmin, - proxyImages: defaultProxyImages, - proxyImageUrl: defaultProxyImageUrl, + proxyHTTPClientTimeout: defaultProxyHTTPClientTimeout, + proxyOption: defaultProxyOption, + proxyMediaTypes: []string{defaultProxyMediaTypes}, + proxyUrl: defaultProxyUrl, fetchYouTubeWatchTime: defaultFetchYouTubeWatchTime, oauth2UserCreationAllowed: defaultOAuth2UserCreation, oauth2ClientID: defaultOAuth2ClientID, @@ -195,6 +203,7 @@ func NewOptions() *Options { httpClientMaxBodySize: defaultHTTPClientMaxBodySize * 1024 * 1024, httpClientProxy: defaultHTTPClientProxy, httpClientUserAgent: defaultHTTPClientUserAgent, + httpServerTimeout: defaultHTTPServerTimeout, authProxyHeader: defaultAuthProxyHeader, authProxyUserCreation: defaultAuthProxyUserCreation, maintenanceMode: defaultMaintenanceMode, @@ -414,14 +423,24 @@ func (o *Options) FetchYouTubeWatchTime() bool { return o.fetchYouTubeWatchTime } -// ProxyImages returns "none" to never proxy, "http-only" to proxy non-HTTPS, "all" to always proxy. -func (o *Options) ProxyImages() string { - return o.proxyImages +// ProxyOption returns "none" to never proxy, "http-only" to proxy non-HTTPS, "all" to always proxy. +func (o *Options) ProxyOption() string { + return o.proxyOption } -// ProxyImageUrl returns a string of a URL to use to proxy image requests -func (o *Options) ProxyImageUrl() string { - return o.proxyImageUrl +// ProxyMediaTypes returns a slice of media types to proxy. +func (o *Options) ProxyMediaTypes() []string { + return o.proxyMediaTypes +} + +// ProxyUrl returns a string of a URL to use to proxy image requests +func (o *Options) ProxyUrl() string { + return o.proxyUrl +} + +// ProxyHTTPClientTimeout returns the time limit in seconds before the proxy HTTP client cancel the request. +func (o *Options) ProxyHTTPClientTimeout() int { + return o.proxyHTTPClientTimeout } // HasHTTPService returns true if the HTTP service is enabled. @@ -457,6 +476,11 @@ func (o *Options) HTTPClientProxy() string { return o.httpClientProxy } +// HTTPServerTimeout returns the time limit in seconds before the HTTP server cancel the request. +func (o *Options) HTTPServerTimeout() int { + return o.httpServerTimeout +} + // HasHTTPClientProxyConfigured returns true if the HTTP proxy is configured. func (o *Options) HasHTTPClientProxyConfigured() bool { return o.httpClientProxy != "" @@ -541,6 +565,7 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option { "HTTP_CLIENT_PROXY": o.httpClientProxy, "HTTP_CLIENT_TIMEOUT": o.httpClientTimeout, "HTTP_CLIENT_USER_AGENT": o.httpClientUserAgent, + "HTTP_SERVER_TIMEOUT": o.httpServerTimeout, "HTTP_SERVICE": o.httpService, "KEY_FILE": o.certKeyFile, "INVIDIOUS_INSTANCE": o.invidiousInstance, @@ -561,9 +586,11 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option { "POLLING_FREQUENCY": o.pollingFrequency, "POLLING_PARSING_ERROR_LIMIT": o.pollingParsingErrorLimit, "POLLING_SCHEDULER": o.pollingScheduler, - "PROXY_IMAGES": o.proxyImages, - "PROXY_IMAGE_URL": o.proxyImageUrl, + "PROXY_HTTP_CLIENT_TIMEOUT": o.proxyHTTPClientTimeout, "PROXY_PRIVATE_KEY": redactSecretValue(string(o.proxyPrivateKey), redactSecret), + "PROXY_MEDIA_TYPES": o.proxyMediaTypes, + "PROXY_OPTION": o.proxyOption, + "PROXY_URL": o.proxyUrl, "ROOT_URL": o.rootURL, "RUN_MIGRATIONS": o.runMigrations, "SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL": o.schedulerEntryFrequencyMaxInterval, diff --git a/config/parser.go b/config/parser.go index 7687a91f..0e3afdf7 100644 --- a/config/parser.go +++ b/config/parser.go @@ -138,10 +138,21 @@ func (p *Parser) parseLines(lines []string) (err error) { p.opts.schedulerEntryFrequencyMinInterval = parseInt(value, defaultSchedulerEntryFrequencyMinInterval) case "POLLING_PARSING_ERROR_LIMIT": p.opts.pollingParsingErrorLimit = parseInt(value, defaultPollingParsingErrorLimit) + // kept for compatibility purpose case "PROXY_IMAGES": - p.opts.proxyImages = parseString(value, defaultProxyImages) + p.opts.proxyOption = parseString(value, defaultProxyOption) + p.opts.proxyMediaTypes = append(p.opts.proxyMediaTypes, "image") + case "PROXY_HTTP_CLIENT_TIMEOUT": + p.opts.proxyHTTPClientTimeout = parseInt(value, defaultProxyHTTPClientTimeout) + case "PROXY_OPTION": + p.opts.proxyOption = parseString(value, defaultProxyOption) + case "PROXY_MEDIA_TYPES": + p.opts.proxyMediaTypes = parseStringList(value, []string{defaultProxyMediaTypes}) + // kept for compatibility purpose case "PROXY_IMAGE_URL": - p.opts.proxyImageUrl = parseString(value, defaultProxyImageUrl) + p.opts.proxyUrl = parseString(value, defaultProxyUrl) + case "PROXY_URL": + p.opts.proxyUrl = parseString(value, defaultProxyUrl) case "CREATE_ADMIN": p.opts.createAdmin = parseBool(value, defaultCreateAdmin) case "ADMIN_USERNAME": @@ -180,6 +191,8 @@ func (p *Parser) parseLines(lines []string) (err error) { p.opts.httpClientProxy = parseString(value, defaultHTTPClientProxy) case "HTTP_CLIENT_USER_AGENT": p.opts.httpClientUserAgent = parseString(value, defaultHTTPClientUserAgent) + case "HTTP_SERVER_TIMEOUT": + p.opts.httpServerTimeout = parseInt(value, defaultHTTPServerTimeout) case "AUTH_PROXY_HEADER": p.opts.authProxyHeader = parseString(value, defaultAuthProxyHeader) case "AUTH_PROXY_USER_CREATION": diff --git a/fever/handler.go b/fever/handler.go index 57fcb184..ce5919c0 100644 --- a/fever/handler.go +++ b/fever/handler.go @@ -310,7 +310,7 @@ func (h *handler) handleItems(w http.ResponseWriter, r *http.Request) { FeedID: entry.FeedID, Title: entry.Title, Author: entry.Author, - HTML: proxy.AbsoluteImageProxyRewriter(h.router, r.Host, entry.Content), + HTML: proxy.AbsoluteProxyRewriter(h.router, r.Host, entry.Content), URL: entry.URL, IsSaved: isSaved, IsRead: isRead, diff --git a/googlereader/handler.go b/googlereader/handler.go index 7bc3f054..4e5272ac 100644 --- a/googlereader/handler.go +++ b/googlereader/handler.go @@ -841,12 +841,17 @@ func (h *handler) streamItemContents(w http.ResponseWriter, r *http.Request) { categories = append(categories, userStarred) } - entry.Content = proxy.AbsoluteImageProxyRewriter(h.router, r.Host, entry.Content) - proxyImage := config.Opts.ProxyImages() + entry.Content = proxy.AbsoluteProxyRewriter(h.router, r.Host, entry.Content) + proxyOption := config.Opts.ProxyOption() for i := range entry.Enclosures { - if strings.HasPrefix(entry.Enclosures[i].MimeType, "image/") && (proxyImage == "all" || proxyImage != "none" && !url.IsHTTPS(entry.Enclosures[i].URL)) { - entry.Enclosures[i].URL = proxy.AbsoluteProxifyURL(h.router, r.Host, entry.Enclosures[i].URL) + if proxyOption == "all" || proxyOption != "none" && !url.IsHTTPS(entry.Enclosures[i].URL) { + for _, mediaType := range config.Opts.ProxyMediaTypes() { + if strings.HasPrefix(entry.Enclosures[i].MimeType, mediaType+"/") { + entry.Enclosures[i].URL = proxy.AbsoluteProxifyURL(h.router, r.Host, entry.Enclosures[i].URL) + break + } + } } } diff --git a/http/response/builder.go b/http/response/builder.go index 21c0cae3..99197293 100644 --- a/http/response/builder.go +++ b/http/response/builder.go @@ -12,6 +12,8 @@ import ( "net/http" "strings" "time" + + "miniflux.app/logger" ) const compressionThreshold = 1024 @@ -88,7 +90,10 @@ func (b *Builder) Write() { case io.Reader: // Compression not implemented in this case b.writeHeaders() - io.Copy(b.w, v) + _, err := io.Copy(b.w, v) + if err != nil { + logger.Error("%v", err) + } } } diff --git a/http/response/html/html.go b/http/response/html/html.go index f529d4aa..c7bf1faf 100644 --- a/http/response/html/html.go +++ b/http/response/html/html.go @@ -72,3 +72,16 @@ func NotFound(w http.ResponseWriter, r *http.Request) { func Redirect(w http.ResponseWriter, r *http.Request, uri string) { http.Redirect(w, r, uri, http.StatusFound) } + +// RequestedRangeNotSatisfiable sends a range not satisfiable error to the client. +func RequestedRangeNotSatisfiable(w http.ResponseWriter, r *http.Request, contentRange string) { + logger.Error("[HTTP:Range Not Satisfiable] %s", r.URL) + + builder := response.New(w, r) + builder.WithStatus(http.StatusRequestedRangeNotSatisfiable) + builder.WithHeader("Content-Type", "text/html; charset=utf-8") + builder.WithHeader("Cache-Control", "no-cache, max-age=0, must-revalidate, no-store") + builder.WithHeader("Content-Range", contentRange) + builder.WithBody("Range Not Satisfiable") + builder.Write() +} diff --git a/http/response/html/html_test.go b/http/response/html/html_test.go index 086935d2..62c9bb80 100644 --- a/http/response/html/html_test.go +++ b/http/response/html/html_test.go @@ -210,3 +210,32 @@ func TestRedirectResponse(t *testing.T) { t.Fatalf(`Unexpected redirect location, got %q instead of %q`, actualResult, expectedResult) } } + +func TestRequestedRangeNotSatisfiable(t *testing.T) { + r, err := http.NewRequest("GET", "/", nil) + if err != nil { + t.Fatal(err) + } + + w := httptest.NewRecorder() + + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + RequestedRangeNotSatisfiable(w, r, "bytes */12777") + }) + + handler.ServeHTTP(w, r) + + resp := w.Result() + defer resp.Body.Close() + + expectedStatusCode := http.StatusRequestedRangeNotSatisfiable + if resp.StatusCode != expectedStatusCode { + t.Fatalf(`Unexpected status code, got %d instead of %d`, resp.StatusCode, expectedStatusCode) + } + + expectedContentRangeHeader := "bytes */12777" + actualContentRangeHeader := resp.Header.Get("Content-Range") + if actualContentRangeHeader != expectedContentRangeHeader { + t.Fatalf(`Unexpected content range header, got %q instead of %q`, actualContentRangeHeader, expectedContentRangeHeader) + } +} diff --git a/miniflux.1 b/miniflux.1 index d900d3bf..05b435f6 100644 --- a/miniflux.1 +++ b/miniflux.1 @@ -365,13 +365,23 @@ Path to a secret key exposed as a file, it should contain $POCKET_CONSUMER_KEY v .br Default is empty\&. .TP -.B PROXY_IMAGES -Avoids mixed content warnings for external images: http-only, all, or none\&. +.B PROXY_OPTION +Avoids mixed content warnings for external media: http-only, all, or none\&. .br Default is http-only\&. .TP -.B PROXY_IMAGE_URL -Sets a server to proxy images through\&. +.B PROXY_MEDIA_TYPES +A list of media types to proxify (comma-separated values): image, audio, video\&. +.br +Default is image only\&. +.TP +.B PROXY_HTTP_CLIENT_TIMEOUT +Time limit in seconds before the proxy HTTP client cancel the request\&. +.br +Default is 120 seconds\&. +.TP +.B PROXY_URL +Sets a server to proxy media through\&. .br Default is empty, miniflux does the proxying\&. .TP @@ -397,6 +407,11 @@ When empty, Miniflux uses a default User-Agent that includes the Miniflux versio .br Default is empty. .TP +.B HTTP_SERVER_TIMEOUT +Time limit in seconds before the HTTP client cancel the request\&. +.br +Default is 300 seconds\&. +.TP .B AUTH_PROXY_HEADER Proxy authentication HTTP header\&. .br diff --git a/proxy/image_proxy.go b/proxy/image_proxy.go deleted file mode 100644 index 581a824e..00000000 --- a/proxy/image_proxy.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2020 Frédéric Guillot. All rights reserved. -// Use of this source code is governed by the Apache 2.0 -// license that can be found in the LICENSE file. - -package proxy // import "miniflux.app/proxy" - -import ( - "strings" - - "miniflux.app/config" - "miniflux.app/reader/sanitizer" - "miniflux.app/url" - - "github.com/PuerkitoBio/goquery" - "github.com/gorilla/mux" -) - -type urlProxyRewriter func(router *mux.Router, url string) string - -// ImageProxyRewriter replaces image URLs with internal proxy URLs. -func ImageProxyRewriter(router *mux.Router, data string) string { - return genericImageProxyRewriter(router, ProxifyURL, data) -} - -// AbsoluteImageProxyRewriter do the same as ImageProxyRewriter except it uses absolute URLs. -func AbsoluteImageProxyRewriter(router *mux.Router, host, data string) string { - proxifyFunction := func(router *mux.Router, url string) string { - return AbsoluteProxifyURL(router, host, url) - } - return genericImageProxyRewriter(router, proxifyFunction, data) -} - -func genericImageProxyRewriter(router *mux.Router, proxifyFunction urlProxyRewriter, data string) string { - proxyImages := config.Opts.ProxyImages() - if proxyImages == "none" { - return data - } - - doc, err := goquery.NewDocumentFromReader(strings.NewReader(data)) - if err != nil { - return data - } - - doc.Find("img").Each(func(i int, img *goquery.Selection) { - if srcAttrValue, ok := img.Attr("src"); ok { - if !isDataURL(srcAttrValue) && (proxyImages == "all" || !url.IsHTTPS(srcAttrValue)) { - img.SetAttr("src", proxifyFunction(router, srcAttrValue)) - } - } - - if srcsetAttrValue, ok := img.Attr("srcset"); ok { - proxifySourceSet(img, router, proxifyFunction, proxyImages, srcsetAttrValue) - } - }) - - doc.Find("picture source").Each(func(i int, sourceElement *goquery.Selection) { - if srcsetAttrValue, ok := sourceElement.Attr("srcset"); ok { - proxifySourceSet(sourceElement, router, proxifyFunction, proxyImages, srcsetAttrValue) - } - }) - - output, err := doc.Find("body").First().Html() - if err != nil { - return data - } - - return output -} - -func proxifySourceSet(element *goquery.Selection, router *mux.Router, proxifyFunction urlProxyRewriter, proxyImages, srcsetAttrValue string) { - imageCandidates := sanitizer.ParseSrcSetAttribute(srcsetAttrValue) - - for _, imageCandidate := range imageCandidates { - if !isDataURL(imageCandidate.ImageURL) && (proxyImages == "all" || !url.IsHTTPS(imageCandidate.ImageURL)) { - imageCandidate.ImageURL = proxifyFunction(router, imageCandidate.ImageURL) - } - } - - element.SetAttr("srcset", imageCandidates.String()) -} - -func isDataURL(s string) bool { - return strings.HasPrefix(s, "data:") -} diff --git a/proxy/media_proxy.go b/proxy/media_proxy.go new file mode 100644 index 00000000..965ce993 --- /dev/null +++ b/proxy/media_proxy.go @@ -0,0 +1,123 @@ +// Copyright 2020 Frédéric Guillot. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +package proxy // import "miniflux.app/proxy" + +import ( + "strings" + + "miniflux.app/config" + "miniflux.app/reader/sanitizer" + "miniflux.app/url" + + "github.com/PuerkitoBio/goquery" + "github.com/gorilla/mux" +) + +type urlProxyRewriter func(router *mux.Router, url string) string + +// ProxyRewriter replaces media URLs with internal proxy URLs. +func ProxyRewriter(router *mux.Router, data string) string { + return genericProxyRewriter(router, ProxifyURL, data) +} + +// AbsoluteProxyRewriter do the same as ProxyRewriter except it uses absolute URLs. +func AbsoluteProxyRewriter(router *mux.Router, host, data string) string { + proxifyFunction := func(router *mux.Router, url string) string { + return AbsoluteProxifyURL(router, host, url) + } + return genericProxyRewriter(router, proxifyFunction, data) +} + +func genericProxyRewriter(router *mux.Router, proxifyFunction urlProxyRewriter, data string) string { + proxyOption := config.Opts.ProxyOption() + if proxyOption == "none" { + return data + } + + doc, err := goquery.NewDocumentFromReader(strings.NewReader(data)) + if err != nil { + return data + } + + for _, mediaType := range config.Opts.ProxyMediaTypes() { + switch mediaType { + case "image": + doc.Find("img").Each(func(i int, img *goquery.Selection) { + if srcAttrValue, ok := img.Attr("src"); ok { + if !isDataURL(srcAttrValue) && (proxyOption == "all" || !url.IsHTTPS(srcAttrValue)) { + img.SetAttr("src", proxifyFunction(router, srcAttrValue)) + } + } + + if srcsetAttrValue, ok := img.Attr("srcset"); ok { + proxifySourceSet(img, router, proxifyFunction, proxyOption, srcsetAttrValue) + } + }) + + doc.Find("picture source").Each(func(i int, sourceElement *goquery.Selection) { + if srcsetAttrValue, ok := sourceElement.Attr("srcset"); ok { + proxifySourceSet(sourceElement, router, proxifyFunction, proxyOption, srcsetAttrValue) + } + }) + + case "audio": + doc.Find("audio").Each(func(i int, audio *goquery.Selection) { + if srcAttrValue, ok := audio.Attr("src"); ok { + if !isDataURL(srcAttrValue) && (proxyOption == "all" || !url.IsHTTPS(srcAttrValue)) { + audio.SetAttr("src", proxifyFunction(router, srcAttrValue)) + } + } + }) + + doc.Find("audio source").Each(func(i int, sourceElement *goquery.Selection) { + if srcAttrValue, ok := sourceElement.Attr("src"); ok { + if !isDataURL(srcAttrValue) && (proxyOption == "all" || !url.IsHTTPS(srcAttrValue)) { + sourceElement.SetAttr("src", proxifyFunction(router, srcAttrValue)) + } + } + }) + + case "video": + doc.Find("video").Each(func(i int, video *goquery.Selection) { + if srcAttrValue, ok := video.Attr("src"); ok { + if !isDataURL(srcAttrValue) && (proxyOption == "all" || !url.IsHTTPS(srcAttrValue)) { + video.SetAttr("src", proxifyFunction(router, srcAttrValue)) + } + } + }) + + doc.Find("video source").Each(func(i int, sourceElement *goquery.Selection) { + if srcAttrValue, ok := sourceElement.Attr("src"); ok { + if !isDataURL(srcAttrValue) && (proxyOption == "all" || !url.IsHTTPS(srcAttrValue)) { + sourceElement.SetAttr("src", proxifyFunction(router, srcAttrValue)) + } + } + }) + } + } + + output, err := doc.Find("body").First().Html() + if err != nil { + return data + } + + return output +} + +func proxifySourceSet(element *goquery.Selection, router *mux.Router, proxifyFunction urlProxyRewriter, proxyOption, srcsetAttrValue string) { + imageCandidates := sanitizer.ParseSrcSetAttribute(srcsetAttrValue) + + for _, imageCandidate := range imageCandidates { + if !isDataURL(imageCandidate.ImageURL) && (proxyOption == "all" || !url.IsHTTPS(imageCandidate.ImageURL)) { + imageCandidate.ImageURL = proxifyFunction(router, imageCandidate.ImageURL) + } + } + + element.SetAttr("srcset", imageCandidates.String()) +} + +func isDataURL(s string) bool { + return strings.HasPrefix(s, "data:") +} diff --git a/proxy/image_proxy_test.go b/proxy/media_proxy_test.go similarity index 57% rename from proxy/image_proxy_test.go rename to proxy/media_proxy_test.go index 2e0a9513..45469f0b 100644 --- a/proxy/image_proxy_test.go +++ b/proxy/media_proxy_test.go @@ -15,7 +15,9 @@ import ( func TestProxyFilterWithHttpDefault(t *testing.T) { os.Clearenv() - os.Setenv("PROXY_IMAGES", "http-only") + os.Setenv("PROXY_OPTION", "http-only") + os.Setenv("PROXY_MEDIA_TYPES", "image") + os.Setenv("PROXY_PRIVATE_KEY", "test") var err error parser := config.NewParser() @@ -25,11 +27,11 @@ func TestProxyFilterWithHttpDefault(t *testing.T) { } r := mux.NewRouter() - r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") + r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `

Test

` - output := ImageProxyRewriter(r, input) - expected := `

Test

` + output := ProxyRewriter(r, input) + expected := `

Test

` if expected != output { t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected) @@ -38,7 +40,8 @@ func TestProxyFilterWithHttpDefault(t *testing.T) { func TestProxyFilterWithHttpsDefault(t *testing.T) { os.Clearenv() - os.Setenv("PROXY_IMAGES", "http-only") + os.Setenv("PROXY_OPTION", "http-only") + os.Setenv("PROXY_MEDIA_TYPES", "image") var err error parser := config.NewParser() @@ -48,10 +51,10 @@ func TestProxyFilterWithHttpsDefault(t *testing.T) { } r := mux.NewRouter() - r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") + r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `

Test

` - output := ImageProxyRewriter(r, input) + output := ProxyRewriter(r, input) expected := `

Test

` if expected != output { @@ -61,7 +64,7 @@ func TestProxyFilterWithHttpsDefault(t *testing.T) { func TestProxyFilterWithHttpNever(t *testing.T) { os.Clearenv() - os.Setenv("PROXY_IMAGES", "none") + os.Setenv("PROXY_OPTION", "none") var err error parser := config.NewParser() @@ -71,10 +74,10 @@ func TestProxyFilterWithHttpNever(t *testing.T) { } r := mux.NewRouter() - r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") + r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `

Test

` - output := ImageProxyRewriter(r, input) + output := ProxyRewriter(r, input) expected := input if expected != output { @@ -84,7 +87,7 @@ func TestProxyFilterWithHttpNever(t *testing.T) { func TestProxyFilterWithHttpsNever(t *testing.T) { os.Clearenv() - os.Setenv("PROXY_IMAGES", "none") + os.Setenv("PROXY_OPTION", "none") var err error parser := config.NewParser() @@ -94,10 +97,10 @@ func TestProxyFilterWithHttpsNever(t *testing.T) { } r := mux.NewRouter() - r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") + r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `

Test

` - output := ImageProxyRewriter(r, input) + output := ProxyRewriter(r, input) expected := input if expected != output { @@ -107,7 +110,9 @@ func TestProxyFilterWithHttpsNever(t *testing.T) { func TestProxyFilterWithHttpAlways(t *testing.T) { os.Clearenv() - os.Setenv("PROXY_IMAGES", "all") + os.Setenv("PROXY_OPTION", "all") + os.Setenv("PROXY_MEDIA_TYPES", "image") + os.Setenv("PROXY_PRIVATE_KEY", "test") var err error parser := config.NewParser() @@ -117,11 +122,11 @@ func TestProxyFilterWithHttpAlways(t *testing.T) { } r := mux.NewRouter() - r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") + r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `

Test

` - output := ImageProxyRewriter(r, input) - expected := `

Test

` + output := ProxyRewriter(r, input) + expected := `

Test

` if expected != output { t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected) @@ -130,7 +135,9 @@ func TestProxyFilterWithHttpAlways(t *testing.T) { func TestProxyFilterWithHttpsAlways(t *testing.T) { os.Clearenv() - os.Setenv("PROXY_IMAGES", "all") + os.Setenv("PROXY_OPTION", "all") + os.Setenv("PROXY_MEDIA_TYPES", "image") + os.Setenv("PROXY_PRIVATE_KEY", "test") var err error parser := config.NewParser() @@ -140,11 +147,11 @@ func TestProxyFilterWithHttpsAlways(t *testing.T) { } r := mux.NewRouter() - r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") + r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `

Test

` - output := ImageProxyRewriter(r, input) - expected := `

Test

` + output := ProxyRewriter(r, input) + expected := `

Test

` if expected != output { t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected) @@ -153,8 +160,9 @@ func TestProxyFilterWithHttpsAlways(t *testing.T) { func TestProxyFilterWithHttpsAlwaysAndCustomProxyServer(t *testing.T) { os.Clearenv() - os.Setenv("PROXY_IMAGES", "all") - os.Setenv("PROXY_IMAGE_URL", "https://proxy-example/proxy") + os.Setenv("PROXY_OPTION", "all") + os.Setenv("PROXY_MEDIA_TYPES", "image") + os.Setenv("PROXY_URL", "https://proxy-example/proxy") var err error parser := config.NewParser() @@ -164,10 +172,10 @@ func TestProxyFilterWithHttpsAlwaysAndCustomProxyServer(t *testing.T) { } r := mux.NewRouter() - r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") + r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `

Test

` - output := ImageProxyRewriter(r, input) + output := ProxyRewriter(r, input) expected := `

Test

` if expected != output { @@ -177,7 +185,8 @@ func TestProxyFilterWithHttpsAlwaysAndCustomProxyServer(t *testing.T) { func TestProxyFilterWithHttpInvalid(t *testing.T) { os.Clearenv() - os.Setenv("PROXY_IMAGES", "invalid") + os.Setenv("PROXY_OPTION", "invalid") + os.Setenv("PROXY_PRIVATE_KEY", "test") var err error parser := config.NewParser() @@ -187,11 +196,11 @@ func TestProxyFilterWithHttpInvalid(t *testing.T) { } r := mux.NewRouter() - r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") + r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `

Test

` - output := ImageProxyRewriter(r, input) - expected := `

Test

` + output := ProxyRewriter(r, input) + expected := `

Test

` if expected != output { t.Errorf(`Not expected output: got "%s" instead of "%s"`, output, expected) @@ -200,7 +209,8 @@ func TestProxyFilterWithHttpInvalid(t *testing.T) { func TestProxyFilterWithHttpsInvalid(t *testing.T) { os.Clearenv() - os.Setenv("PROXY_IMAGES", "invalid") + os.Setenv("PROXY_OPTION", "invalid") + os.Setenv("PROXY_PRIVATE_KEY", "test") var err error parser := config.NewParser() @@ -210,10 +220,10 @@ func TestProxyFilterWithHttpsInvalid(t *testing.T) { } r := mux.NewRouter() - r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") + r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `

Test

` - output := ImageProxyRewriter(r, input) + output := ProxyRewriter(r, input) expected := `

Test

` if expected != output { @@ -223,7 +233,9 @@ func TestProxyFilterWithHttpsInvalid(t *testing.T) { func TestProxyFilterWithSrcset(t *testing.T) { os.Clearenv() - os.Setenv("PROXY_IMAGES", "all") + os.Setenv("PROXY_OPTION", "all") + os.Setenv("PROXY_MEDIA_TYPES", "image") + os.Setenv("PROXY_PRIVATE_KEY", "test") var err error parser := config.NewParser() @@ -233,11 +245,11 @@ func TestProxyFilterWithSrcset(t *testing.T) { } r := mux.NewRouter() - r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") + r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `

test

` - expected := `

test

` - output := ImageProxyRewriter(r, input) + expected := `

test

` + output := ProxyRewriter(r, input) if expected != output { t.Errorf(`Not expected output: got %s`, output) @@ -246,7 +258,9 @@ func TestProxyFilterWithSrcset(t *testing.T) { func TestProxyFilterWithEmptySrcset(t *testing.T) { os.Clearenv() - os.Setenv("PROXY_IMAGES", "all") + os.Setenv("PROXY_OPTION", "all") + os.Setenv("PROXY_MEDIA_TYPES", "image") + os.Setenv("PROXY_PRIVATE_KEY", "test") var err error parser := config.NewParser() @@ -256,11 +270,11 @@ func TestProxyFilterWithEmptySrcset(t *testing.T) { } r := mux.NewRouter() - r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") + r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `

test

` - expected := `

test

` - output := ImageProxyRewriter(r, input) + expected := `

test

` + output := ProxyRewriter(r, input) if expected != output { t.Errorf(`Not expected output: got %s`, output) @@ -269,7 +283,9 @@ func TestProxyFilterWithEmptySrcset(t *testing.T) { func TestProxyFilterWithPictureSource(t *testing.T) { os.Clearenv() - os.Setenv("PROXY_IMAGES", "all") + os.Setenv("PROXY_OPTION", "all") + os.Setenv("PROXY_MEDIA_TYPES", "image") + os.Setenv("PROXY_PRIVATE_KEY", "test") var err error parser := config.NewParser() @@ -279,11 +295,11 @@ func TestProxyFilterWithPictureSource(t *testing.T) { } r := mux.NewRouter() - r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") + r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `` - expected := `` - output := ImageProxyRewriter(r, input) + expected := `` + output := ProxyRewriter(r, input) if expected != output { t.Errorf(`Not expected output: got %s`, output) @@ -292,7 +308,9 @@ func TestProxyFilterWithPictureSource(t *testing.T) { func TestProxyFilterOnlyNonHTTPWithPictureSource(t *testing.T) { os.Clearenv() - os.Setenv("PROXY_IMAGES", "https") + os.Setenv("PROXY_OPTION", "https") + os.Setenv("PROXY_MEDIA_TYPES", "image") + os.Setenv("PROXY_PRIVATE_KEY", "test") var err error parser := config.NewParser() @@ -302,20 +320,21 @@ func TestProxyFilterOnlyNonHTTPWithPictureSource(t *testing.T) { } r := mux.NewRouter() - r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") + r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `` - expected := `` - output := ImageProxyRewriter(r, input) + expected := `` + output := ProxyRewriter(r, input) if expected != output { t.Errorf(`Not expected output: got %s`, output) } } -func TestImageProxyWithImageDataURL(t *testing.T) { +func TestProxyWithImageDataURL(t *testing.T) { os.Clearenv() - os.Setenv("PROXY_IMAGES", "all") + os.Setenv("PROXY_OPTION", "all") + os.Setenv("PROXY_MEDIA_TYPES", "image") var err error parser := config.NewParser() @@ -325,20 +344,21 @@ func TestImageProxyWithImageDataURL(t *testing.T) { } r := mux.NewRouter() - r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") + r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `` expected := `` - output := ImageProxyRewriter(r, input) + output := ProxyRewriter(r, input) if expected != output { t.Errorf(`Not expected output: got %s`, output) } } -func TestImageProxyWithImageSourceDataURL(t *testing.T) { +func TestProxyWithImageSourceDataURL(t *testing.T) { os.Clearenv() - os.Setenv("PROXY_IMAGES", "all") + os.Setenv("PROXY_OPTION", "all") + os.Setenv("PROXY_MEDIA_TYPES", "image") var err error parser := config.NewParser() @@ -348,11 +368,11 @@ func TestImageProxyWithImageSourceDataURL(t *testing.T) { } r := mux.NewRouter() - r.HandleFunc("/proxy/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") + r.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", func(w http.ResponseWriter, r *http.Request) {}).Name("proxy") input := `` expected := `` - output := ImageProxyRewriter(r, input) + output := ProxyRewriter(r, input) if expected != output { t.Errorf(`Not expected output: got %s`, output) diff --git a/proxy/proxy.go b/proxy/proxy.go index 21e9b2e4..1fe9eceb 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -21,7 +21,7 @@ import ( // ProxifyURL generates a relative URL for a proxified resource. func ProxifyURL(router *mux.Router, link string) string { if link != "" { - proxyImageUrl := config.Opts.ProxyImageUrl() + proxyImageUrl := config.Opts.ProxyUrl() if proxyImageUrl == "" { mac := hmac.New(sha256.New, config.Opts.ProxyPrivateKey()) @@ -44,7 +44,7 @@ func ProxifyURL(router *mux.Router, link string) string { // AbsoluteProxifyURL generates an absolute URL for a proxified resource. func AbsoluteProxifyURL(router *mux.Router, host, link string) string { if link != "" { - proxyImageUrl := config.Opts.ProxyImageUrl() + proxyImageUrl := config.Opts.ProxyUrl() if proxyImageUrl == "" { mac := hmac.New(sha256.New, config.Opts.ProxyPrivateKey()) diff --git a/service/httpd/httpd.go b/service/httpd/httpd.go index a0bcab30..7cbed4bd 100644 --- a/service/httpd/httpd.go +++ b/service/httpd/httpd.go @@ -37,9 +37,9 @@ func Serve(store *storage.Storage, pool *worker.Pool) *http.Server { certDomain := config.Opts.CertDomain() listenAddr := config.Opts.ListenAddr() server := &http.Server{ - ReadTimeout: 300 * time.Second, - WriteTimeout: 300 * time.Second, - IdleTimeout: 300 * time.Second, + ReadTimeout: time.Duration(config.Opts.HTTPServerTimeout()) * time.Second, + WriteTimeout: time.Duration(config.Opts.HTTPServerTimeout()) * time.Second, + IdleTimeout: time.Duration(config.Opts.HTTPServerTimeout()) * time.Second, Handler: setupHandler(store, pool), } diff --git a/template/functions.go b/template/functions.go index 92b482e9..fd2b44a7 100644 --- a/template/functions.go +++ b/template/functions.go @@ -61,17 +61,25 @@ func (f *funcMap) Map() template.FuncMap { return template.HTML(str) }, "proxyFilter": func(data string) string { - return proxy.ImageProxyRewriter(f.router, data) + return proxy.ProxyRewriter(f.router, data) }, "proxyURL": func(link string) string { - proxyImages := config.Opts.ProxyImages() + proxyOption := config.Opts.ProxyOption() - if proxyImages == "all" || (proxyImages != "none" && !url.IsHTTPS(link)) { + if proxyOption == "all" || (proxyOption != "none" && !url.IsHTTPS(link)) { return proxy.ProxifyURL(f.router, link) } return link }, + "mustBeProxyfied": func(mediaType string) bool { + for _, t := range config.Opts.ProxyMediaTypes() { + if t == mediaType { + return true + } + } + return false + }, "domain": func(websiteURL string) string { return url.Domain(websiteURL) }, diff --git a/template/templates/views/entry.html b/template/templates/views/entry.html index 150041c7..1cc17039 100644 --- a/template/templates/views/entry.html +++ b/template/templates/views/entry.html @@ -159,18 +159,26 @@ {{ if hasPrefix .MimeType "audio/" }}
{{ else if hasPrefix .MimeType "video/" }}
{{ else if hasPrefix .MimeType "image/" }}
- {{ if $.user }} + {{ if (and $.user (mustBeProxyfied "image")) }} {{ .URL }} ({{ .MimeType }}) {{ else }} {{ .URL }} ({{ .MimeType }}) diff --git a/ui/entry_scraper.go b/ui/entry_scraper.go index e556dc94..77755c74 100644 --- a/ui/entry_scraper.go +++ b/ui/entry_scraper.go @@ -67,5 +67,5 @@ func (h *handler) fetchContent(w http.ResponseWriter, r *http.Request) { readingTime := locale.NewPrinter(user.Language).Plural("entry.estimated_reading_time", entry.ReadingTime, entry.ReadingTime) - json.OK(w, r, map[string]string{"content": proxy.ImageProxyRewriter(h.router, entry.Content), "reading_time": readingTime}) + json.OK(w, r, map[string]string{"content": proxy.ProxyRewriter(h.router, entry.Content), "reading_time": readingTime}) } diff --git a/ui/proxy.go b/ui/proxy.go index 6f43086b..5dd50697 100644 --- a/ui/proxy.go +++ b/ui/proxy.go @@ -20,8 +20,8 @@ import ( "miniflux.app/logger" ) -func (h *handler) imageProxy(w http.ResponseWriter, r *http.Request) { - // If we receive a "If-None-Match" header, we assume the image is already stored in browser cache. +func (h *handler) mediaProxy(w http.ResponseWriter, r *http.Request) { + // If we receive a "If-None-Match" header, we assume the media is already stored in browser cache. if r.Header.Get("If-None-Match") != "" { w.WriteHeader(http.StatusNotModified) return @@ -55,10 +55,10 @@ func (h *handler) imageProxy(w http.ResponseWriter, r *http.Request) { return } - imageURL := string(decodedURL) - logger.Debug(`[Proxy] Fetching %q`, imageURL) + mediaURL := string(decodedURL) + logger.Debug(`[Proxy] Fetching %q`, mediaURL) - req, err := http.NewRequest("GET", imageURL, nil) + req, err := http.NewRequest("GET", mediaURL, nil) if err != nil { html.ServerError(w, r, err) return @@ -67,8 +67,18 @@ func (h *handler) imageProxy(w http.ResponseWriter, r *http.Request) { // Note: User-Agent HTTP header is omitted to avoid being blocked by bot protection mechanisms. req.Header.Add("Connection", "close") + forwardedRequestHeader := []string{"Range", "Accept", "Accept-Encoding"} + for _, requestHeaderName := range forwardedRequestHeader { + if r.Header.Get(requestHeaderName) != "" { + req.Header.Add(requestHeaderName, r.Header.Get(requestHeaderName)) + } + } + clt := &http.Client{ - Timeout: time.Duration(config.Opts.HTTPClientTimeout()) * time.Second, + Transport: &http.Transport{ + IdleConnTimeout: time.Duration(config.Opts.ProxyHTTPClientTimeout()) * time.Second, + }, + Timeout: time.Duration(config.Opts.ProxyHTTPClientTimeout()) * time.Second, } resp, err := clt.Do(req) @@ -78,8 +88,13 @@ func (h *handler) imageProxy(w http.ResponseWriter, r *http.Request) { } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - logger.Error(`[Proxy] Status Code is %d for URL %q`, resp.StatusCode, imageURL) + if resp.StatusCode == http.StatusRequestedRangeNotSatisfiable { + logger.Error(`[Proxy] Status Code is %d for URL %q`, resp.StatusCode, mediaURL) + html.RequestedRangeNotSatisfiable(w, r, resp.Header.Get("Content-Range")) + return + } + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent { + logger.Error(`[Proxy] Status Code is %d for URL %q`, resp.StatusCode, mediaURL) html.NotFound(w, r) return } @@ -87,8 +102,15 @@ func (h *handler) imageProxy(w http.ResponseWriter, r *http.Request) { etag := crypto.HashFromBytes(decodedURL) response.New(w, r).WithCaching(etag, 72*time.Hour, func(b *response.Builder) { + b.WithStatus(resp.StatusCode) b.WithHeader("Content-Security-Policy", `default-src 'self'`) b.WithHeader("Content-Type", resp.Header.Get("Content-Type")) + forwardedResponseHeader := []string{"Content-Encoding", "Content-Type", "Content-Length", "Accept-Ranges", "Content-Range"} + for _, responseHeaderName := range forwardedResponseHeader { + if resp.Header.Get(responseHeaderName) != "" { + b.WithHeader(responseHeaderName, resp.Header.Get(responseHeaderName)) + } + } b.WithBody(resp.Body) b.WithoutCompression() b.Write() diff --git a/ui/ui.go b/ui/ui.go index 3cac810e..15f5da78 100644 --- a/ui/ui.go +++ b/ui/ui.go @@ -96,7 +96,7 @@ func Serve(router *mux.Router, store *storage.Storage, pool *worker.Pool) { uiRouter.HandleFunc("/entry/status", handler.updateEntriesStatus).Name("updateEntriesStatus").Methods(http.MethodPost) uiRouter.HandleFunc("/entry/save/{entryID}", handler.saveEntry).Name("saveEntry").Methods(http.MethodPost) uiRouter.HandleFunc("/entry/download/{entryID}", handler.fetchContent).Name("fetchContent").Methods(http.MethodPost) - uiRouter.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", handler.imageProxy).Name("proxy").Methods(http.MethodGet) + uiRouter.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", handler.mediaProxy).Name("proxy").Methods(http.MethodGet) uiRouter.HandleFunc("/entry/bookmark/{entryID}", handler.toggleBookmark).Name("toggleBookmark").Methods(http.MethodPost) // Share pages.