From 3f14d08095bd320ff0b74bd742d4f4050bcf4011 Mon Sep 17 00:00:00 2001 From: Romain de Laage Date: Sat, 15 Oct 2022 08:17:17 +0200 Subject: [PATCH] Proxify images in API responses --- api/api.go | 7 ++++--- api/entry.go | 25 +++++++++++++++++++++---- config/options.go | 12 ++++++++++++ config/parser.go | 13 +++++++++++++ fever/handler.go | 8 +++++--- googlereader/handler.go | 11 +++++++++++ miniflux.1 | 5 +++++ proxy/image_proxy.go | 24 +++++++++++++++++++----- proxy/proxy.go | 37 +++++++++++++++++++++++++++++++++++-- ui/middleware.go | 3 ++- ui/proxy.go | 18 ++++++++++++++++++ ui/ui.go | 2 +- 12 files changed, 146 insertions(+), 19 deletions(-) diff --git a/api/api.go b/api/api.go index 9d44aacd..ceab2697 100644 --- a/api/api.go +++ b/api/api.go @@ -14,13 +14,14 @@ import ( ) type handler struct { - store *storage.Storage - pool *worker.Pool + store *storage.Storage + pool *worker.Pool + router *mux.Router } // Serve declares API routes for the application. func Serve(router *mux.Router, store *storage.Storage, pool *worker.Pool) { - handler := &handler{store, pool} + handler := &handler{store, pool, router} sr := router.PathPrefix("/v1").Subrouter() middleware := newMiddleware(store) diff --git a/api/entry.go b/api/entry.go index f773bfbf..839f6cac 100644 --- a/api/entry.go +++ b/api/entry.go @@ -9,17 +9,21 @@ import ( "errors" "net/http" "strconv" + "strings" "time" + "miniflux.app/config" "miniflux.app/http/request" "miniflux.app/http/response/json" "miniflux.app/model" + "miniflux.app/proxy" "miniflux.app/reader/processor" "miniflux.app/storage" + "miniflux.app/url" "miniflux.app/validator" ) -func getEntryFromBuilder(w http.ResponseWriter, r *http.Request, b *storage.EntryQueryBuilder) { +func (h *handler) getEntryFromBuilder(w http.ResponseWriter, r *http.Request, b *storage.EntryQueryBuilder) { entry, err := b.GetEntry() if err != nil { json.ServerError(w, r, err) @@ -31,6 +35,15 @@ func getEntryFromBuilder(w http.ResponseWriter, r *http.Request, b *storage.Entr return } + entry.Content = proxy.AbsoluteImageProxyRewriter(h.router, r.Host, entry.Content) + proxyImage := config.Opts.ProxyImages() + + for i := range entry.Enclosures { + if strings.HasPrefix(entry.Enclosures[i].MimeType, "image/") && (proxyImage == "all" || proxyImage != "none" && !url.IsHTTPS(entry.Enclosures[i].URL)) { + entry.Enclosures[i].URL = proxy.AbsoluteProxifyURL(h.router, r.Host, entry.Enclosures[i].URL) + } + } + json.OK(w, r, entry) } @@ -42,7 +55,7 @@ func (h *handler) getFeedEntry(w http.ResponseWriter, r *http.Request) { builder.WithFeedID(feedID) builder.WithEntryID(entryID) - getEntryFromBuilder(w, r, builder) + h.getEntryFromBuilder(w, r, builder) } func (h *handler) getCategoryEntry(w http.ResponseWriter, r *http.Request) { @@ -53,7 +66,7 @@ func (h *handler) getCategoryEntry(w http.ResponseWriter, r *http.Request) { builder.WithCategoryID(categoryID) builder.WithEntryID(entryID) - getEntryFromBuilder(w, r, builder) + h.getEntryFromBuilder(w, r, builder) } func (h *handler) getEntry(w http.ResponseWriter, r *http.Request) { @@ -61,7 +74,7 @@ func (h *handler) getEntry(w http.ResponseWriter, r *http.Request) { builder := h.store.NewEntryQueryBuilder(request.UserID(r)) builder.WithEntryID(entryID) - getEntryFromBuilder(w, r, builder) + h.getEntryFromBuilder(w, r, builder) } func (h *handler) getFeedEntries(w http.ResponseWriter, r *http.Request) { @@ -141,6 +154,10 @@ func (h *handler) findEntries(w http.ResponseWriter, r *http.Request, feedID int return } + for i := range entries { + entries[i].Content = proxy.AbsoluteImageProxyRewriter(h.router, r.Host, entries[i].Content) + } + json.OK(w, r, &entriesResponse{Total: count, Entries: entries}) } diff --git a/config/options.go b/config/options.go index 4af429f1..44af9a3e 100644 --- a/config/options.go +++ b/config/options.go @@ -5,6 +5,7 @@ package config // import "miniflux.app/config" import ( + "crypto/rand" "fmt" "sort" "strings" @@ -139,10 +140,14 @@ type Options struct { metricsAllowedNetworks []string watchdog bool invidiousInstance string + proxyPrivateKey []byte } // NewOptions returns Options with default values. func NewOptions() *Options { + randomKey := make([]byte, 16) + rand.Read(randomKey) + return &Options{ HTTPS: defaultHTTPS, logDateTime: defaultLogDateTime, @@ -199,6 +204,7 @@ func NewOptions() *Options { metricsAllowedNetworks: []string{defaultMetricsAllowedNetworks}, watchdog: defaultWatchdog, invidiousInstance: defaultInvidiousInstance, + proxyPrivateKey: randomKey, } } @@ -498,6 +504,11 @@ func (o *Options) InvidiousInstance() string { return o.invidiousInstance } +// ProxyPrivateKey returns the private key used by the media proxy +func (o *Options) ProxyPrivateKey() []byte { + return o.proxyPrivateKey +} + // SortedOptions returns options as a list of key value pairs, sorted by keys. func (o *Options) SortedOptions(redactSecret bool) []*Option { var keyValues = map[string]interface{}{ @@ -552,6 +563,7 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option { "POLLING_SCHEDULER": o.pollingScheduler, "PROXY_IMAGES": o.proxyImages, "PROXY_IMAGE_URL": o.proxyImageUrl, + "PROXY_PRIVATE_KEY": redactSecretValue(string(o.proxyPrivateKey), redactSecret), "ROOT_URL": o.rootURL, "RUN_MIGRATIONS": o.runMigrations, "SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL": o.schedulerEntryFrequencyMaxInterval, diff --git a/config/parser.go b/config/parser.go index 310e9323..7687a91f 100644 --- a/config/parser.go +++ b/config/parser.go @@ -7,6 +7,7 @@ package config // import "miniflux.app/config" import ( "bufio" "bytes" + "crypto/rand" "errors" "fmt" "io" @@ -199,6 +200,10 @@ func (p *Parser) parseLines(lines []string) (err error) { p.opts.watchdog = parseBool(value, defaultWatchdog) case "INVIDIOUS_INSTANCE": p.opts.invidiousInstance = parseString(value, defaultInvidiousInstance) + case "PROXY_PRIVATE_KEY": + randomKey := make([]byte, 16) + rand.Read(randomKey) + p.opts.proxyPrivateKey = parseBytes(value, randomKey) } } @@ -279,6 +284,14 @@ func parseStringList(value string, fallback []string) []string { return strList } +func parseBytes(value string, fallback []byte) []byte { + if value == "" { + return fallback + } + + return []byte(value) +} + func readSecretFile(filename, fallback string) string { data, err := os.ReadFile(filename) if err != nil { diff --git a/fever/handler.go b/fever/handler.go index 9508358c..57fcb184 100644 --- a/fever/handler.go +++ b/fever/handler.go @@ -15,6 +15,7 @@ import ( "miniflux.app/integration" "miniflux.app/logger" "miniflux.app/model" + "miniflux.app/proxy" "miniflux.app/storage" "github.com/gorilla/mux" @@ -22,7 +23,7 @@ import ( // Serve handles Fever API calls. func Serve(router *mux.Router, store *storage.Storage) { - handler := &handler{store} + handler := &handler{store, router} sr := router.PathPrefix("/fever").Subrouter() sr.Use(newMiddleware(store).serve) @@ -30,7 +31,8 @@ func Serve(router *mux.Router, store *storage.Storage) { } type handler struct { - store *storage.Storage + store *storage.Storage + router *mux.Router } func (h *handler) serve(w http.ResponseWriter, r *http.Request) { @@ -308,7 +310,7 @@ func (h *handler) handleItems(w http.ResponseWriter, r *http.Request) { FeedID: entry.FeedID, Title: entry.Title, Author: entry.Author, - HTML: entry.Content, + HTML: proxy.AbsoluteImageProxyRewriter(h.router, r.Host, entry.Content), URL: entry.URL, IsSaved: isSaved, IsRead: isRead, diff --git a/googlereader/handler.go b/googlereader/handler.go index c55925e6..7bc3f054 100644 --- a/googlereader/handler.go +++ b/googlereader/handler.go @@ -21,9 +21,11 @@ import ( "miniflux.app/integration" "miniflux.app/logger" "miniflux.app/model" + "miniflux.app/proxy" mff "miniflux.app/reader/handler" mfs "miniflux.app/reader/subscription" "miniflux.app/storage" + "miniflux.app/url" "miniflux.app/validator" ) @@ -839,6 +841,15 @@ func (h *handler) streamItemContents(w http.ResponseWriter, r *http.Request) { categories = append(categories, userStarred) } + entry.Content = proxy.AbsoluteImageProxyRewriter(h.router, r.Host, entry.Content) + proxyImage := config.Opts.ProxyImages() + + for i := range entry.Enclosures { + if strings.HasPrefix(entry.Enclosures[i].MimeType, "image/") && (proxyImage == "all" || proxyImage != "none" && !url.IsHTTPS(entry.Enclosures[i].URL)) { + entry.Enclosures[i].URL = proxy.AbsoluteProxifyURL(h.router, r.Host, entry.Enclosures[i].URL) + } + } + contentItems[i] = contentItem{ ID: fmt.Sprintf(EntryIDLong, entry.ID), Title: entry.Title, diff --git a/miniflux.1 b/miniflux.1 index e6777978..d900d3bf 100644 --- a/miniflux.1 +++ b/miniflux.1 @@ -426,6 +426,11 @@ Enabled by default\&. Set a custom invidious instance to use\&. .br Default is yewtu.be\&. +.TP +.B PROXY_PRIVATE_KEY +Set a custom custom private key used to sign proxified media url\&. +.br +Default is randomly generated at startup\&. .SH AUTHORS .P diff --git a/proxy/image_proxy.go b/proxy/image_proxy.go index e486654e..581a824e 100644 --- a/proxy/image_proxy.go +++ b/proxy/image_proxy.go @@ -15,8 +15,22 @@ import ( "github.com/gorilla/mux" ) +type urlProxyRewriter func(router *mux.Router, url string) string + // ImageProxyRewriter replaces image URLs with internal proxy URLs. func ImageProxyRewriter(router *mux.Router, data string) string { + return genericImageProxyRewriter(router, ProxifyURL, data) +} + +// AbsoluteImageProxyRewriter do the same as ImageProxyRewriter except it uses absolute URLs. +func AbsoluteImageProxyRewriter(router *mux.Router, host, data string) string { + proxifyFunction := func(router *mux.Router, url string) string { + return AbsoluteProxifyURL(router, host, url) + } + return genericImageProxyRewriter(router, proxifyFunction, data) +} + +func genericImageProxyRewriter(router *mux.Router, proxifyFunction urlProxyRewriter, data string) string { proxyImages := config.Opts.ProxyImages() if proxyImages == "none" { return data @@ -30,18 +44,18 @@ func ImageProxyRewriter(router *mux.Router, data string) string { doc.Find("img").Each(func(i int, img *goquery.Selection) { if srcAttrValue, ok := img.Attr("src"); ok { if !isDataURL(srcAttrValue) && (proxyImages == "all" || !url.IsHTTPS(srcAttrValue)) { - img.SetAttr("src", ProxifyURL(router, srcAttrValue)) + img.SetAttr("src", proxifyFunction(router, srcAttrValue)) } } if srcsetAttrValue, ok := img.Attr("srcset"); ok { - proxifySourceSet(img, router, proxyImages, srcsetAttrValue) + proxifySourceSet(img, router, proxifyFunction, proxyImages, srcsetAttrValue) } }) doc.Find("picture source").Each(func(i int, sourceElement *goquery.Selection) { if srcsetAttrValue, ok := sourceElement.Attr("srcset"); ok { - proxifySourceSet(sourceElement, router, proxyImages, srcsetAttrValue) + proxifySourceSet(sourceElement, router, proxifyFunction, proxyImages, srcsetAttrValue) } }) @@ -53,12 +67,12 @@ func ImageProxyRewriter(router *mux.Router, data string) string { return output } -func proxifySourceSet(element *goquery.Selection, router *mux.Router, proxyImages, srcsetAttrValue string) { +func proxifySourceSet(element *goquery.Selection, router *mux.Router, proxifyFunction urlProxyRewriter, proxyImages, srcsetAttrValue string) { imageCandidates := sanitizer.ParseSrcSetAttribute(srcsetAttrValue) for _, imageCandidate := range imageCandidates { if !isDataURL(imageCandidate.ImageURL) && (proxyImages == "all" || !url.IsHTTPS(imageCandidate.ImageURL)) { - imageCandidate.ImageURL = ProxifyURL(router, imageCandidate.ImageURL) + imageCandidate.ImageURL = proxifyFunction(router, imageCandidate.ImageURL) } } diff --git a/proxy/proxy.go b/proxy/proxy.go index 312891b4..21e9b2e4 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -5,6 +5,8 @@ package proxy // import "miniflux.app/proxy" import ( + "crypto/hmac" + "crypto/sha256" "encoding/base64" "net/url" "path" @@ -16,13 +18,44 @@ import ( "miniflux.app/config" ) -// ProxifyURL generates an URL for a proxified resource. +// ProxifyURL generates a relative URL for a proxified resource. func ProxifyURL(router *mux.Router, link string) string { if link != "" { proxyImageUrl := config.Opts.ProxyImageUrl() if proxyImageUrl == "" { - return route.Path(router, "proxy", "encodedURL", base64.URLEncoding.EncodeToString([]byte(link))) + mac := hmac.New(sha256.New, config.Opts.ProxyPrivateKey()) + mac.Write([]byte(link)) + digest := mac.Sum(nil) + return route.Path(router, "proxy", "encodedDigest", base64.URLEncoding.EncodeToString(digest), "encodedURL", base64.URLEncoding.EncodeToString([]byte(link))) + } + + proxyUrl, err := url.Parse(proxyImageUrl) + if err != nil { + return "" + } + + proxyUrl.Path = path.Join(proxyUrl.Path, base64.URLEncoding.EncodeToString([]byte(link))) + return proxyUrl.String() + } + return "" +} + +// AbsoluteProxifyURL generates an absolute URL for a proxified resource. +func AbsoluteProxifyURL(router *mux.Router, host, link string) string { + if link != "" { + proxyImageUrl := config.Opts.ProxyImageUrl() + + if proxyImageUrl == "" { + mac := hmac.New(sha256.New, config.Opts.ProxyPrivateKey()) + mac.Write([]byte(link)) + digest := mac.Sum(nil) + path := route.Path(router, "proxy", "encodedDigest", base64.URLEncoding.EncodeToString(digest), "encodedURL", base64.URLEncoding.EncodeToString([]byte(link))) + if config.Opts.HTTPS { + return "https://" + host + path + } else { + return "http://" + host + path + } } proxyUrl, err := url.Parse(proxyImageUrl) diff --git a/ui/middleware.go b/ui/middleware.go index 21af40af..31a912ab 100644 --- a/ui/middleware.go +++ b/ui/middleware.go @@ -143,7 +143,8 @@ func (m *middleware) isPublicRoute(r *http.Request) bool { "robots", "sharedEntry", "healthcheck", - "offline": + "offline", + "proxy": return true default: return false diff --git a/ui/proxy.go b/ui/proxy.go index cd175852..6f43086b 100644 --- a/ui/proxy.go +++ b/ui/proxy.go @@ -5,6 +5,8 @@ package ui // import "miniflux.app/ui" import ( + "crypto/hmac" + "crypto/sha256" "encoding/base64" "errors" "net/http" @@ -25,18 +27,34 @@ func (h *handler) imageProxy(w http.ResponseWriter, r *http.Request) { return } + encodedDigest := request.RouteStringParam(r, "encodedDigest") encodedURL := request.RouteStringParam(r, "encodedURL") if encodedURL == "" { html.BadRequest(w, r, errors.New("No URL provided")) return } + decodedDigest, err := base64.URLEncoding.DecodeString(encodedDigest) + if err != nil { + html.BadRequest(w, r, errors.New("Unable to decode this Digest")) + return + } + decodedURL, err := base64.URLEncoding.DecodeString(encodedURL) if err != nil { html.BadRequest(w, r, errors.New("Unable to decode this URL")) return } + mac := hmac.New(sha256.New, config.Opts.ProxyPrivateKey()) + mac.Write(decodedURL) + expectedMAC := mac.Sum(nil) + + if !hmac.Equal(decodedDigest, expectedMAC) { + html.Forbidden(w, r) + return + } + imageURL := string(decodedURL) logger.Debug(`[Proxy] Fetching %q`, imageURL) diff --git a/ui/ui.go b/ui/ui.go index ee694561..21ab45ba 100644 --- a/ui/ui.go +++ b/ui/ui.go @@ -94,7 +94,7 @@ func Serve(router *mux.Router, store *storage.Storage, pool *worker.Pool) { uiRouter.HandleFunc("/entry/status", handler.updateEntriesStatus).Name("updateEntriesStatus").Methods(http.MethodPost) uiRouter.HandleFunc("/entry/save/{entryID}", handler.saveEntry).Name("saveEntry").Methods(http.MethodPost) uiRouter.HandleFunc("/entry/download/{entryID}", handler.fetchContent).Name("fetchContent").Methods(http.MethodPost) - uiRouter.HandleFunc("/proxy/{encodedURL}", handler.imageProxy).Name("proxy").Methods(http.MethodGet) + uiRouter.HandleFunc("/proxy/{encodedDigest}/{encodedURL}", handler.imageProxy).Name("proxy").Methods(http.MethodGet) uiRouter.HandleFunc("/entry/bookmark/{entryID}", handler.toggleBookmark).Name("toggleBookmark").Methods(http.MethodPost) // Share pages.