mirror of https://github.com/miniflux/v2.git
Refactor icon finder
Changes: - Continue the discovery process when the feed icon is invalid - Search all icons from the HTML document and do not stop on the first one
This commit is contained in:
parent
7650c81ad9
commit
9fd2dfa680
|
@ -241,8 +241,8 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64, forceRefresh bool
|
||||||
|
|
||||||
func checkFeedIcon(store *storage.Storage, feedID int64, websiteURL, feedIconURL, userAgent string, fetchViaProxy, allowSelfSignedCertificates bool) {
|
func checkFeedIcon(store *storage.Storage, feedID int64, websiteURL, feedIconURL, userAgent string, fetchViaProxy, allowSelfSignedCertificates bool) {
|
||||||
if !store.HasIcon(feedID) {
|
if !store.HasIcon(feedID) {
|
||||||
icon, err := icon.FindIcon(websiteURL, feedIconURL, userAgent, fetchViaProxy, allowSelfSignedCertificates)
|
iconFinder := icon.NewIconFinder(websiteURL, feedIconURL, userAgent, fetchViaProxy, allowSelfSignedCertificates)
|
||||||
if err != nil {
|
if icon, err := iconFinder.FindIcon(); err != nil {
|
||||||
slog.Warn("Unable to find feed icon",
|
slog.Warn("Unable to find feed icon",
|
||||||
slog.Int64("feed_id", feedID),
|
slog.Int64("feed_id", feedID),
|
||||||
slog.String("website_url", websiteURL),
|
slog.String("website_url", websiteURL),
|
||||||
|
|
|
@ -7,6 +7,7 @@ import (
|
||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"log/slog"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
@ -19,106 +20,170 @@ import (
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// FindIcon try to find the website's icon.
|
type IconFinder struct {
|
||||||
func FindIcon(websiteURL, feedIconURL, userAgent string, fetchViaProxy, allowSelfSignedCertificates bool) (icon *model.Icon, err error) {
|
websiteURL string
|
||||||
if feedIconURL == "" {
|
feedIconURL string
|
||||||
feedIconURL, err = fetchHTMLDocumentAndFindIconURL(websiteURL, userAgent, fetchViaProxy, allowSelfSignedCertificates)
|
userAgent string
|
||||||
|
fetchViaProxy bool
|
||||||
|
allowSelfSignedCertificates bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewIconFinder(websiteURL, feedIconURL, userAgent string, fetchViaProxy, allowSelfSignedCertificates bool) *IconFinder {
|
||||||
|
return &IconFinder{
|
||||||
|
websiteURL: websiteURL,
|
||||||
|
feedIconURL: feedIconURL,
|
||||||
|
userAgent: userAgent,
|
||||||
|
fetchViaProxy: fetchViaProxy,
|
||||||
|
allowSelfSignedCertificates: allowSelfSignedCertificates,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *IconFinder) FindIcon() (*model.Icon, error) {
|
||||||
|
slog.Debug("Begin icon discovery process",
|
||||||
|
slog.String("website_url", f.websiteURL),
|
||||||
|
slog.String("feed_icon_url", f.feedIconURL),
|
||||||
|
)
|
||||||
|
|
||||||
|
if f.feedIconURL != "" {
|
||||||
|
if icon, err := f.FetchFeedIcon(); err != nil {
|
||||||
|
slog.Debug("Unable to download icon from feed",
|
||||||
|
slog.String("website_url", f.websiteURL),
|
||||||
|
slog.String("feed_icon_url", f.feedIconURL),
|
||||||
|
slog.Any("error", err),
|
||||||
|
)
|
||||||
|
} else if icon != nil {
|
||||||
|
return icon, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if icon, err := f.FetchIconsFromHTMLDocument(); err != nil {
|
||||||
|
slog.Debug("Unable to fetch icons from HTML document",
|
||||||
|
slog.String("website_url", f.websiteURL),
|
||||||
|
slog.Any("error", err),
|
||||||
|
)
|
||||||
|
} else if icon != nil {
|
||||||
|
return icon, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return f.FetchDefaultIcon()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *IconFinder) FetchDefaultIcon() (*model.Icon, error) {
|
||||||
|
slog.Debug("Fetching default icon",
|
||||||
|
slog.String("website_url", f.websiteURL),
|
||||||
|
)
|
||||||
|
|
||||||
|
iconURL, err := urllib.JoinBaseURLAndPath(urllib.RootURL(f.websiteURL), "favicon.ico")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, fmt.Errorf(`icon: unable to join root URL and path: %w`, err)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if strings.HasPrefix(feedIconURL, "data:") {
|
icon, err := f.DownloadIcon(iconURL)
|
||||||
return parseImageDataURL(feedIconURL)
|
|
||||||
}
|
|
||||||
|
|
||||||
feedIconURL, err = generateIconURL(websiteURL, feedIconURL)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if icon, err = downloadIcon(feedIconURL, userAgent, fetchViaProxy, allowSelfSignedCertificates); err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return icon, nil
|
return icon, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func generateIconURL(websiteURL, feedIconURL string) (iconURL string, err error) {
|
func (f *IconFinder) FetchFeedIcon() (*model.Icon, error) {
|
||||||
feedIconURL = strings.TrimSpace(feedIconURL)
|
slog.Debug("Fetching feed icon",
|
||||||
|
slog.String("website_url", f.websiteURL),
|
||||||
|
slog.String("feed_icon_url", f.feedIconURL),
|
||||||
|
)
|
||||||
|
|
||||||
if feedIconURL == "" {
|
iconURL, err := urllib.AbsoluteURL(f.websiteURL, f.feedIconURL)
|
||||||
iconURL, err = urllib.JoinBaseURLAndPath(urllib.RootURL(websiteURL), "favicon.ico")
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf(`icon: unable to join base URL and path: %w`, err)
|
return nil, fmt.Errorf(`icon: unable to convert icon URL to absolute URL: %w`, err)
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
iconURL, err = urllib.AbsoluteURL(websiteURL, feedIconURL)
|
return f.DownloadIcon(iconURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *IconFinder) FetchIconsFromHTMLDocument() (*model.Icon, error) {
|
||||||
|
slog.Debug("Searching icons from HTML document",
|
||||||
|
slog.String("website_url", f.websiteURL),
|
||||||
|
)
|
||||||
|
|
||||||
|
documentBody, err := f.FetchRootDocument()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf(`icon: unable to convert icon URL to absolute URL: %w`, err)
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
iconURLs, err := findIconURLsFromHTMLDocument(documentBody)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
slog.Debug("Searched icon from HTML document",
|
||||||
|
slog.String("website_url", f.websiteURL),
|
||||||
|
slog.String("icon_urls", strings.Join(iconURLs, ",")),
|
||||||
|
)
|
||||||
|
|
||||||
|
for _, iconURL := range iconURLs {
|
||||||
|
if strings.HasPrefix(iconURL, "data:") {
|
||||||
|
slog.Debug("Found icon with data URL",
|
||||||
|
slog.String("website_url", f.websiteURL),
|
||||||
|
)
|
||||||
|
return parseImageDataURL(iconURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
iconURL, err = urllib.AbsoluteURL(f.websiteURL, iconURL)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf(`icon: unable to convert icon URL to absolute URL: %w`, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if icon, err := f.DownloadIcon(iconURL); err != nil {
|
||||||
|
slog.Debug("Unable to download icon from HTML document",
|
||||||
|
slog.String("website_url", f.websiteURL),
|
||||||
|
slog.String("icon_url", iconURL),
|
||||||
|
slog.Any("error", err),
|
||||||
|
)
|
||||||
|
} else if icon != nil {
|
||||||
|
slog.Debug("Found icon from HTML document",
|
||||||
|
slog.String("website_url", f.websiteURL),
|
||||||
|
slog.String("icon_url", iconURL),
|
||||||
|
)
|
||||||
|
return icon, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return iconURL, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func fetchHTMLDocumentAndFindIconURL(websiteURL, userAgent string, fetchViaProxy, allowSelfSignedCertificates bool) (string, error) {
|
func (f *IconFinder) FetchRootDocument() (io.Reader, error) {
|
||||||
rootURL := urllib.RootURL(websiteURL)
|
rootURL := urllib.RootURL(f.websiteURL)
|
||||||
|
|
||||||
clt := client.NewClientWithConfig(rootURL, config.Opts)
|
clt := client.NewClientWithConfig(rootURL, config.Opts)
|
||||||
clt.WithUserAgent(userAgent)
|
clt.WithUserAgent(f.userAgent)
|
||||||
clt.AllowSelfSignedCertificates = allowSelfSignedCertificates
|
clt.AllowSelfSignedCertificates = f.allowSelfSignedCertificates
|
||||||
|
|
||||||
if fetchViaProxy {
|
if f.fetchViaProxy {
|
||||||
clt.WithProxy()
|
clt.WithProxy()
|
||||||
}
|
}
|
||||||
|
|
||||||
response, err := clt.Get()
|
response, err := clt.Get()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("icon: unable to download website index page: %v", err)
|
return nil, fmt.Errorf("icon: unable to download website index page: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if response.HasServerFailure() {
|
if response.HasServerFailure() {
|
||||||
return "", fmt.Errorf("icon: unable to download website index page: status=%d", response.StatusCode)
|
return nil, fmt.Errorf("icon: unable to download website index page: status=%d", response.StatusCode)
|
||||||
}
|
}
|
||||||
|
|
||||||
return findIconURLFromHTMLDocument(response.Body)
|
return response.Body, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func findIconURLFromHTMLDocument(body io.Reader) (string, error) {
|
func (f *IconFinder) DownloadIcon(iconURL string) (*model.Icon, error) {
|
||||||
queries := []string{
|
slog.Debug("Downloading icon",
|
||||||
"link[rel='shortcut icon']",
|
slog.String("website_url", f.websiteURL),
|
||||||
"link[rel='Shortcut Icon']",
|
slog.String("icon_url", iconURL),
|
||||||
"link[rel='icon shortcut']",
|
)
|
||||||
"link[rel='icon']",
|
|
||||||
}
|
|
||||||
|
|
||||||
doc, err := goquery.NewDocumentFromReader(body)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("icon: unable to read document: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
var iconURL string
|
|
||||||
for _, query := range queries {
|
|
||||||
doc.Find(query).Each(func(i int, s *goquery.Selection) {
|
|
||||||
if href, exists := s.Attr("href"); exists {
|
|
||||||
iconURL = strings.TrimSpace(href)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
|
|
||||||
if iconURL != "" {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return iconURL, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func downloadIcon(iconURL, userAgent string, fetchViaProxy, allowSelfSignedCertificates bool) (*model.Icon, error) {
|
|
||||||
clt := client.NewClientWithConfig(iconURL, config.Opts)
|
clt := client.NewClientWithConfig(iconURL, config.Opts)
|
||||||
clt.WithUserAgent(userAgent)
|
clt.WithUserAgent(f.userAgent)
|
||||||
clt.AllowSelfSignedCertificates = allowSelfSignedCertificates
|
clt.AllowSelfSignedCertificates = f.allowSelfSignedCertificates
|
||||||
if fetchViaProxy {
|
if f.fetchViaProxy {
|
||||||
clt.WithProxy()
|
clt.WithProxy()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -149,6 +214,43 @@ func downloadIcon(iconURL, userAgent string, fetchViaProxy, allowSelfSignedCerti
|
||||||
return icon, nil
|
return icon, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func findIconURLsFromHTMLDocument(body io.Reader) ([]string, error) {
|
||||||
|
queries := []string{
|
||||||
|
"link[rel='shortcut icon']",
|
||||||
|
"link[rel='Shortcut Icon']",
|
||||||
|
"link[rel='icon shortcut']",
|
||||||
|
"link[rel='icon']",
|
||||||
|
}
|
||||||
|
|
||||||
|
doc, err := goquery.NewDocumentFromReader(body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("icon: unable to read document: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var iconURLs []string
|
||||||
|
for _, query := range queries {
|
||||||
|
slog.Debug("Searching icon URL in HTML document", slog.String("query", query))
|
||||||
|
|
||||||
|
doc.Find(query).Each(func(i int, s *goquery.Selection) {
|
||||||
|
var iconURL string
|
||||||
|
|
||||||
|
if href, exists := s.Attr("href"); exists {
|
||||||
|
iconURL = strings.TrimSpace(href)
|
||||||
|
}
|
||||||
|
|
||||||
|
if iconURL != "" {
|
||||||
|
iconURLs = append(iconURLs, iconURL)
|
||||||
|
|
||||||
|
slog.Debug("Found icon URL in HTML document",
|
||||||
|
slog.String("query", query),
|
||||||
|
slog.String("icon_url", iconURL))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return iconURLs, nil
|
||||||
|
}
|
||||||
|
|
||||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs#syntax
|
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs#syntax
|
||||||
// data:[<mediatype>][;base64],<data>
|
// data:[<mediatype>][;base64],<data>
|
||||||
func parseImageDataURL(value string) (*model.Icon, error) {
|
func parseImageDataURL(value string) (*model.Icon, error) {
|
||||||
|
|
|
@ -112,59 +112,16 @@ func TestParseDocumentWithWhitespaceIconURL(t *testing.T) {
|
||||||
/static/img/favicon.ico
|
/static/img/favicon.ico
|
||||||
">`
|
">`
|
||||||
|
|
||||||
iconURL, err := findIconURLFromHTMLDocument(strings.NewReader(html))
|
iconURLs, err := findIconURLsFromHTMLDocument(strings.NewReader(html))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if iconURL != "/static/img/favicon.ico" {
|
if len(iconURLs) != 1 {
|
||||||
t.Errorf(`Invalid icon URL, got %q`, iconURL)
|
t.Fatalf(`Invalid number of icon URLs, got %d`, len(iconURLs))
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGenerateIconURL(t *testing.T) {
|
if iconURLs[0] != "/static/img/favicon.ico" {
|
||||||
iconURL, err := generateIconURL("https://example.org/", "/favicon.png")
|
t.Errorf(`Invalid icon URL, got %q`, iconURLs[0])
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if iconURL != "https://example.org/favicon.png" {
|
|
||||||
t.Errorf(`Invalid icon URL, got %q`, iconURL)
|
|
||||||
}
|
|
||||||
|
|
||||||
iconURL, err = generateIconURL("https://example.org/", "img/favicon.png")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if iconURL != "https://example.org/img/favicon.png" {
|
|
||||||
t.Errorf(`Invalid icon URL, got %q`, iconURL)
|
|
||||||
}
|
|
||||||
|
|
||||||
iconURL, err = generateIconURL("https://example.org/", "https://example.org/img/favicon.png")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if iconURL != "https://example.org/img/favicon.png" {
|
|
||||||
t.Errorf(`Invalid icon URL, got %q`, iconURL)
|
|
||||||
}
|
|
||||||
|
|
||||||
iconURL, err = generateIconURL("https://example.org/", "//example.org/img/favicon.png")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if iconURL != "https://example.org/img/favicon.png" {
|
|
||||||
t.Errorf(`Invalid icon URL, got %q`, iconURL)
|
|
||||||
}
|
|
||||||
|
|
||||||
iconURL, err = generateIconURL("https://example.org/", " ")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if iconURL != "https://example.org/favicon.ico" {
|
|
||||||
t.Errorf(`Invalid icon URL, got %q`, iconURL)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue