mirror of
https://github.com/miniflux/v2.git
synced 2024-08-30 16:17:20 +02:00
453 lines
10 KiB
Go
453 lines
10 KiB
Go
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
|
|
package rss // import "miniflux.app/v2/internal/reader/rss"
|
|
|
|
import (
|
|
"encoding/xml"
|
|
"html"
|
|
"log/slog"
|
|
"path"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"miniflux.app/v2/internal/crypto"
|
|
"miniflux.app/v2/internal/model"
|
|
"miniflux.app/v2/internal/reader/date"
|
|
"miniflux.app/v2/internal/reader/dublincore"
|
|
"miniflux.app/v2/internal/reader/media"
|
|
"miniflux.app/v2/internal/reader/sanitizer"
|
|
"miniflux.app/v2/internal/urllib"
|
|
)
|
|
|
|
// Specs: https://cyber.harvard.edu/rss/rss.html
|
|
type rssFeed struct {
|
|
XMLName xml.Name `xml:"rss"`
|
|
Version string `xml:"version,attr"`
|
|
Title string `xml:"channel>title"`
|
|
Links []rssLink `xml:"channel>link"`
|
|
ImageURL string `xml:"channel>image>url"`
|
|
Language string `xml:"channel>language"`
|
|
Description string `xml:"channel>description"`
|
|
PubDate string `xml:"channel>pubDate"`
|
|
ManagingEditor string `xml:"channel>managingEditor"`
|
|
Webmaster string `xml:"channel>webMaster"`
|
|
TimeToLive rssTTL `xml:"channel>ttl"`
|
|
Items []rssItem `xml:"channel>item"`
|
|
PodcastFeedElement
|
|
}
|
|
|
|
type rssTTL struct {
|
|
Data string `xml:",chardata"`
|
|
}
|
|
|
|
func (r *rssTTL) Value() int {
|
|
if r.Data == "" {
|
|
return 0
|
|
}
|
|
|
|
value, err := strconv.Atoi(r.Data)
|
|
if err != nil {
|
|
return 0
|
|
}
|
|
|
|
return value
|
|
}
|
|
|
|
func (r *rssFeed) Transform(baseURL string) *model.Feed {
|
|
var err error
|
|
|
|
feed := new(model.Feed)
|
|
|
|
siteURL := r.siteURL()
|
|
feed.SiteURL, err = urllib.AbsoluteURL(baseURL, siteURL)
|
|
if err != nil {
|
|
feed.SiteURL = siteURL
|
|
}
|
|
|
|
feedURL := r.feedURL()
|
|
feed.FeedURL, err = urllib.AbsoluteURL(baseURL, feedURL)
|
|
if err != nil {
|
|
feed.FeedURL = feedURL
|
|
}
|
|
|
|
feed.Title = html.UnescapeString(strings.TrimSpace(r.Title))
|
|
if feed.Title == "" {
|
|
feed.Title = feed.SiteURL
|
|
}
|
|
|
|
feed.IconURL = strings.TrimSpace(r.ImageURL)
|
|
feed.TTL = r.TimeToLive.Value()
|
|
|
|
for _, item := range r.Items {
|
|
entry := item.Transform()
|
|
if entry.Author == "" {
|
|
entry.Author = r.feedAuthor()
|
|
}
|
|
|
|
if entry.URL == "" {
|
|
entry.URL = feed.SiteURL
|
|
} else {
|
|
entryURL, err := urllib.AbsoluteURL(feed.SiteURL, entry.URL)
|
|
if err == nil {
|
|
entry.URL = entryURL
|
|
}
|
|
}
|
|
|
|
if entry.Title == "" {
|
|
entry.Title = sanitizer.TruncateHTML(entry.Content, 100)
|
|
}
|
|
|
|
if entry.Title == "" {
|
|
entry.Title = entry.URL
|
|
}
|
|
|
|
feed.Entries = append(feed.Entries, entry)
|
|
}
|
|
|
|
return feed
|
|
}
|
|
|
|
func (r *rssFeed) siteURL() string {
|
|
for _, element := range r.Links {
|
|
if element.XMLName.Space == "" {
|
|
return strings.TrimSpace(element.Data)
|
|
}
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
func (r *rssFeed) feedURL() string {
|
|
for _, element := range r.Links {
|
|
if element.XMLName.Space == "http://www.w3.org/2005/Atom" {
|
|
return strings.TrimSpace(element.Href)
|
|
}
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
func (r rssFeed) feedAuthor() string {
|
|
author := r.PodcastAuthor()
|
|
switch {
|
|
case r.ManagingEditor != "":
|
|
author = r.ManagingEditor
|
|
case r.Webmaster != "":
|
|
author = r.Webmaster
|
|
}
|
|
return sanitizer.StripTags(strings.TrimSpace(author))
|
|
}
|
|
|
|
type rssGUID struct {
|
|
XMLName xml.Name
|
|
Data string `xml:",chardata"`
|
|
IsPermaLink string `xml:"isPermaLink,attr"`
|
|
}
|
|
|
|
type rssLink struct {
|
|
XMLName xml.Name
|
|
Data string `xml:",chardata"`
|
|
Href string `xml:"href,attr"`
|
|
Rel string `xml:"rel,attr"`
|
|
}
|
|
|
|
type rssCommentLink struct {
|
|
XMLName xml.Name
|
|
Data string `xml:",chardata"`
|
|
}
|
|
|
|
type rssAuthor struct {
|
|
XMLName xml.Name
|
|
Data string `xml:",chardata"`
|
|
Name string `xml:"name"`
|
|
Email string `xml:"email"`
|
|
Inner string `xml:",innerxml"`
|
|
}
|
|
|
|
type rssTitle struct {
|
|
XMLName xml.Name
|
|
Data string `xml:",chardata"`
|
|
Inner string `xml:",innerxml"`
|
|
}
|
|
|
|
type rssEnclosure struct {
|
|
URL string `xml:"url,attr"`
|
|
Type string `xml:"type,attr"`
|
|
Length string `xml:"length,attr"`
|
|
}
|
|
|
|
type rssCategory struct {
|
|
XMLName xml.Name
|
|
Data string `xml:",chardata"`
|
|
Inner string `xml:",innerxml"`
|
|
}
|
|
|
|
func (enclosure *rssEnclosure) Size() int64 {
|
|
if enclosure.Length == "" {
|
|
return 0
|
|
}
|
|
size, _ := strconv.ParseInt(enclosure.Length, 10, 0)
|
|
return size
|
|
}
|
|
|
|
type rssItem struct {
|
|
GUID rssGUID `xml:"guid"`
|
|
Title []rssTitle `xml:"title"`
|
|
Links []rssLink `xml:"link"`
|
|
Description string `xml:"description"`
|
|
PubDate string `xml:"pubDate"`
|
|
Authors []rssAuthor `xml:"author"`
|
|
CommentLinks []rssCommentLink `xml:"comments"`
|
|
EnclosureLinks []rssEnclosure `xml:"enclosure"`
|
|
Categories []rssCategory `xml:"category"`
|
|
dublincore.DublinCoreItemElement
|
|
FeedBurnerElement
|
|
PodcastEntryElement
|
|
media.Element
|
|
}
|
|
|
|
func (r *rssItem) Transform() *model.Entry {
|
|
entry := model.NewEntry()
|
|
entry.URL = r.entryURL()
|
|
entry.CommentsURL = r.entryCommentsURL()
|
|
entry.Date = r.entryDate()
|
|
entry.Author = r.entryAuthor()
|
|
entry.Hash = r.entryHash()
|
|
entry.Content = r.entryContent()
|
|
entry.Title = r.entryTitle()
|
|
entry.Enclosures = r.entryEnclosures()
|
|
entry.Tags = r.entryCategories()
|
|
if duration, err := normalizeDuration(r.Duration); err == nil {
|
|
entry.ReadingTime = duration
|
|
}
|
|
|
|
return entry
|
|
}
|
|
|
|
func (r *rssItem) entryDate() time.Time {
|
|
value := r.PubDate
|
|
if r.DublinCoreDate != "" {
|
|
value = r.DublinCoreDate
|
|
}
|
|
|
|
if value != "" {
|
|
result, err := date.Parse(value)
|
|
if err != nil {
|
|
slog.Debug("Unable to parse date from RSS feed",
|
|
slog.String("date", value),
|
|
slog.String("guid", r.GUID.Data),
|
|
slog.Any("error", err),
|
|
)
|
|
return time.Now()
|
|
}
|
|
|
|
return result
|
|
}
|
|
|
|
return time.Now()
|
|
}
|
|
|
|
func (r *rssItem) entryAuthor() string {
|
|
author := ""
|
|
|
|
for _, rssAuthor := range r.Authors {
|
|
switch rssAuthor.XMLName.Space {
|
|
case "http://www.itunes.com/dtds/podcast-1.0.dtd", "http://www.google.com/schemas/play-podcasts/1.0":
|
|
author = rssAuthor.Data
|
|
case "http://www.w3.org/2005/Atom":
|
|
if rssAuthor.Name != "" {
|
|
author = rssAuthor.Name
|
|
} else if rssAuthor.Email != "" {
|
|
author = rssAuthor.Email
|
|
}
|
|
default:
|
|
if rssAuthor.Name != "" {
|
|
author = rssAuthor.Name
|
|
} else if strings.Contains(rssAuthor.Inner, "<![CDATA[") {
|
|
author = rssAuthor.Data
|
|
} else {
|
|
author = rssAuthor.Inner
|
|
}
|
|
}
|
|
}
|
|
|
|
if author == "" {
|
|
author = r.GetSanitizedCreator()
|
|
}
|
|
|
|
return sanitizer.StripTags(strings.TrimSpace(author))
|
|
}
|
|
|
|
func (r *rssItem) entryHash() string {
|
|
for _, value := range []string{r.GUID.Data, r.entryURL()} {
|
|
if value != "" {
|
|
return crypto.Hash(value)
|
|
}
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
func (r *rssItem) entryTitle() string {
|
|
var title string
|
|
|
|
for _, rssTitle := range r.Title {
|
|
switch rssTitle.XMLName.Space {
|
|
case "http://search.yahoo.com/mrss/":
|
|
// Ignore title in media namespace
|
|
case "http://purl.org/dc/elements/1.1/":
|
|
title = rssTitle.Data
|
|
default:
|
|
title = rssTitle.Data
|
|
}
|
|
|
|
if title != "" {
|
|
break
|
|
}
|
|
}
|
|
|
|
return html.UnescapeString(strings.TrimSpace(title))
|
|
}
|
|
|
|
func (r *rssItem) entryContent() string {
|
|
for _, value := range []string{r.DublinCoreContent, r.Description, r.PodcastDescription()} {
|
|
if value != "" {
|
|
return value
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func (r *rssItem) entryURL() string {
|
|
if r.FeedBurnerLink != "" {
|
|
return r.FeedBurnerLink
|
|
}
|
|
|
|
for _, link := range r.Links {
|
|
if link.XMLName.Space == "http://www.w3.org/2005/Atom" && link.Href != "" && isValidLinkRelation(link.Rel) {
|
|
return strings.TrimSpace(link.Href)
|
|
}
|
|
|
|
if link.Data != "" {
|
|
return strings.TrimSpace(link.Data)
|
|
}
|
|
}
|
|
|
|
// Specs: https://cyber.harvard.edu/rss/rss.html#ltguidgtSubelementOfLtitemgt
|
|
// isPermaLink is optional, its default value is true.
|
|
// If its value is false, the guid may not be assumed to be a url, or a url to anything in particular.
|
|
if r.GUID.IsPermaLink == "true" || r.GUID.IsPermaLink == "" {
|
|
return strings.TrimSpace(r.GUID.Data)
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
func (r *rssItem) entryEnclosures() model.EnclosureList {
|
|
enclosures := make(model.EnclosureList, 0)
|
|
duplicates := make(map[string]bool)
|
|
|
|
for _, mediaThumbnail := range r.AllMediaThumbnails() {
|
|
if _, found := duplicates[mediaThumbnail.URL]; !found {
|
|
duplicates[mediaThumbnail.URL] = true
|
|
enclosures = append(enclosures, &model.Enclosure{
|
|
URL: mediaThumbnail.URL,
|
|
MimeType: mediaThumbnail.MimeType(),
|
|
Size: mediaThumbnail.Size(),
|
|
})
|
|
}
|
|
}
|
|
|
|
for _, enclosure := range r.EnclosureLinks {
|
|
enclosureURL := enclosure.URL
|
|
|
|
if r.FeedBurnerEnclosureLink != "" {
|
|
filename := path.Base(r.FeedBurnerEnclosureLink)
|
|
if strings.Contains(enclosureURL, filename) {
|
|
enclosureURL = r.FeedBurnerEnclosureLink
|
|
}
|
|
}
|
|
|
|
if enclosureURL == "" {
|
|
continue
|
|
}
|
|
|
|
if _, found := duplicates[enclosureURL]; !found {
|
|
duplicates[enclosureURL] = true
|
|
|
|
enclosures = append(enclosures, &model.Enclosure{
|
|
URL: enclosureURL,
|
|
MimeType: enclosure.Type,
|
|
Size: enclosure.Size(),
|
|
})
|
|
}
|
|
}
|
|
|
|
for _, mediaContent := range r.AllMediaContents() {
|
|
if _, found := duplicates[mediaContent.URL]; !found {
|
|
duplicates[mediaContent.URL] = true
|
|
enclosures = append(enclosures, &model.Enclosure{
|
|
URL: mediaContent.URL,
|
|
MimeType: mediaContent.MimeType(),
|
|
Size: mediaContent.Size(),
|
|
})
|
|
}
|
|
}
|
|
|
|
for _, mediaPeerLink := range r.AllMediaPeerLinks() {
|
|
if _, found := duplicates[mediaPeerLink.URL]; !found {
|
|
duplicates[mediaPeerLink.URL] = true
|
|
enclosures = append(enclosures, &model.Enclosure{
|
|
URL: mediaPeerLink.URL,
|
|
MimeType: mediaPeerLink.MimeType(),
|
|
Size: mediaPeerLink.Size(),
|
|
})
|
|
}
|
|
}
|
|
|
|
return enclosures
|
|
}
|
|
|
|
func (r *rssItem) entryCategories() []string {
|
|
categoryList := make([]string, 0)
|
|
|
|
for _, rssCategory := range r.Categories {
|
|
if strings.Contains(rssCategory.Inner, "<![CDATA[") {
|
|
categoryList = append(categoryList, strings.TrimSpace(rssCategory.Data))
|
|
} else {
|
|
categoryList = append(categoryList, strings.TrimSpace(rssCategory.Inner))
|
|
}
|
|
}
|
|
|
|
return categoryList
|
|
}
|
|
|
|
func (r *rssItem) entryCommentsURL() string {
|
|
for _, commentLink := range r.CommentLinks {
|
|
if commentLink.XMLName.Space == "" {
|
|
commentsURL := strings.TrimSpace(commentLink.Data)
|
|
// The comments URL is supposed to be absolute (some feeds publishes incorrect comments URL)
|
|
// See https://cyber.harvard.edu/rss/rss.html#ltcommentsgtSubelementOfLtitemgt
|
|
if urllib.IsAbsoluteURL(commentsURL) {
|
|
return commentsURL
|
|
}
|
|
}
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
func isValidLinkRelation(rel string) bool {
|
|
switch rel {
|
|
case "", "alternate", "enclosure", "related", "self", "via":
|
|
return true
|
|
default:
|
|
if strings.HasPrefix(rel, "http") {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
}
|