From f722fd12086240b748ec35600526005feaa762b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Guillot?= Date: Wed, 2 Dec 2020 20:47:11 -0800 Subject: [PATCH] Handle invalid feeds with relative URLs --- reader/atom/atom_03.go | 22 +++-- reader/atom/atom_03_test.go | 14 +-- reader/atom/atom_10.go | 22 +++-- reader/atom/atom_10_test.go | 98 ++++++++++++++------- reader/atom/parser.go | 6 +- reader/feed/handler.go | 4 +- reader/json/json.go | 20 +++-- reader/json/parser.go | 4 +- reader/json/parser_test.go | 22 ++--- reader/parser/parser.go | 10 +-- reader/parser/parser_test.go | 162 +++++++++++++++++++++++++++++++++-- reader/rdf/parser.go | 4 +- reader/rdf/parser_test.go | 28 +++--- reader/rdf/rdf.go | 9 +- reader/rss/parser.go | 4 +- reader/rss/parser_test.go | 84 +++++++++--------- reader/rss/rss.go | 22 +++-- url/url_test.go | 14 +-- 18 files changed, 392 insertions(+), 157 deletions(-) diff --git a/reader/atom/atom_03.go b/reader/atom/atom_03.go index 7a86204b..36fbb12c 100644 --- a/reader/atom/atom_03.go +++ b/reader/atom/atom_03.go @@ -27,12 +27,24 @@ type atom03Feed struct { Entries []atom03Entry `xml:"entry"` } -func (a *atom03Feed) Transform() *model.Feed { - feed := new(model.Feed) - feed.FeedURL = a.Links.firstLinkWithRelation("self") - feed.SiteURL = a.Links.originalLink() - feed.Title = a.Title.String() +func (a *atom03Feed) Transform(baseURL string) *model.Feed { + var err error + feed := new(model.Feed) + + feedURL := a.Links.firstLinkWithRelation("self") + feed.FeedURL, err = url.AbsoluteURL(baseURL, feedURL) + if err != nil { + feed.FeedURL = feedURL + } + + siteURL := a.Links.originalLink() + feed.SiteURL, err = url.AbsoluteURL(baseURL, siteURL) + if err != nil { + feed.SiteURL = siteURL + } + + feed.Title = a.Title.String() if feed.Title == "" { feed.Title = feed.SiteURL } diff --git a/reader/atom/atom_03_test.go b/reader/atom/atom_03_test.go index 063b02c4..75083d93 100644 --- a/reader/atom/atom_03_test.go +++ b/reader/atom/atom_03_test.go @@ -28,7 +28,7 @@ func TestParseAtom03(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://diveintomark.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -37,7 +37,7 @@ func TestParseAtom03(t *testing.T) { t.Errorf("Incorrect title, got: %s", feed.Title) } - if feed.FeedURL != "" { + if feed.FeedURL != "http://diveintomark.org/" { t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL) } @@ -88,7 +88,7 @@ func TestParseAtom03WithoutFeedTitle(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://diveintomark.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -111,7 +111,7 @@ func TestParseAtom03WithoutEntryTitle(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://diveintomark.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -142,7 +142,7 @@ func TestParseAtom03WithSummaryOnly(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://diveintomark.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -173,7 +173,7 @@ func TestParseAtom03WithXMLContent(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://diveintomark.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -204,7 +204,7 @@ func TestParseAtom03WithBase64Content(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://diveintomark.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } diff --git a/reader/atom/atom_10.go b/reader/atom/atom_10.go index e7797d2e..d60c3ce4 100644 --- a/reader/atom/atom_10.go +++ b/reader/atom/atom_10.go @@ -31,12 +31,24 @@ type atom10Feed struct { Entries []atom10Entry `xml:"entry"` } -func (a *atom10Feed) Transform() *model.Feed { - feed := new(model.Feed) - feed.FeedURL = a.Links.firstLinkWithRelation("self") - feed.SiteURL = a.Links.originalLink() - feed.Title = a.Title.String() +func (a *atom10Feed) Transform(baseURL string) *model.Feed { + var err error + feed := new(model.Feed) + + feedURL := a.Links.firstLinkWithRelation("self") + feed.FeedURL, err = url.AbsoluteURL(baseURL, feedURL) + if err != nil { + feed.FeedURL = feedURL + } + + siteURL := a.Links.originalLink() + feed.SiteURL, err = url.AbsoluteURL(baseURL, siteURL) + if err != nil { + feed.SiteURL = siteURL + } + + feed.Title = a.Title.String() if feed.Title == "" { feed.Title = feed.SiteURL } diff --git a/reader/atom/atom_10_test.go b/reader/atom/atom_10_test.go index ad897440..4999aca2 100644 --- a/reader/atom/atom_10_test.go +++ b/reader/atom/atom_10_test.go @@ -32,7 +32,7 @@ func TestParseAtomSample(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://example.org/feed.xml", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -41,7 +41,7 @@ func TestParseAtomSample(t *testing.T) { t.Errorf("Incorrect title, got: %s", feed.Title) } - if feed.FeedURL != "" { + if feed.FeedURL != "http://example.org/feed.xml" { t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL) } @@ -90,7 +90,7 @@ func TestParseFeedWithoutTitle(t *testing.T) { 2003-12-13T18:30:02Z ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -121,7 +121,7 @@ func TestParseEntryWithoutTitle(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -140,7 +140,7 @@ func TestParseFeedURL(t *testing.T) { 2003-12-13T18:30:02Z ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -154,6 +154,42 @@ func TestParseFeedURL(t *testing.T) { } } +func TestParseFeedWithRelativeURL(t *testing.T) { + data := ` + + Example Feed + + + + + Test + + + /blog/article.html + 2003-12-13T18:30:02Z + Some text. + + + ` + + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) + if err != nil { + t.Fatal(err) + } + + if feed.FeedURL != "https://example.org/blog/atom.xml" { + t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL) + } + + if feed.SiteURL != "https://example.org/blog" { + t.Errorf("Incorrect site URL, got: %s", feed.SiteURL) + } + + if feed.Entries[0].URL != "https://example.org/blog/article.html" { + t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL) + } +} + func TestParseEntryWithRelativeURL(t *testing.T) { data := ` @@ -170,7 +206,7 @@ func TestParseEntryWithRelativeURL(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.net/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -198,7 +234,7 @@ func TestParseEntryTitleWithWhitespaces(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -224,7 +260,7 @@ func TestParseEntryTitleWithHTMLAndCDATA(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -250,7 +286,7 @@ func TestParseEntryTitleWithHTML(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -276,7 +312,7 @@ func TestParseEntryTitleWithXHTML(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -302,7 +338,7 @@ func TestParseEntrySummaryWithXHTML(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -328,7 +364,7 @@ func TestParseEntrySummaryWithHTML(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -354,7 +390,7 @@ func TestParseEntrySummaryWithPlainText(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -383,7 +419,7 @@ func TestParseEntryWithAuthorName(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -412,7 +448,7 @@ func TestParseEntryWithoutAuthorName(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -462,7 +498,7 @@ func TestParseEntryWithEnclosures(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -522,7 +558,7 @@ func TestParseEntryWithoutEnclosureURL(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -555,7 +591,7 @@ func TestParseEntryWithPublished(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -581,7 +617,7 @@ func TestParseEntryWithPublishedAndUpdated(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -593,7 +629,7 @@ func TestParseEntryWithPublishedAndUpdated(t *testing.T) { func TestParseInvalidXml(t *testing.T) { data := `garbage` - _, err := Parse(bytes.NewBufferString(data)) + _, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err == nil { t.Error("Parse should returns an error") } @@ -608,7 +644,7 @@ func TestParseTitleWithSingleQuote(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -627,7 +663,7 @@ func TestParseTitleWithEncodedSingleQuote(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -646,7 +682,7 @@ func TestParseTitleWithSingleQuoteAndHTMLType(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -665,7 +701,7 @@ func TestParseWithHTMLEntity(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -684,7 +720,7 @@ func TestParseWithInvalidCharacterEntity(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -717,7 +753,7 @@ A website: http://example.org/ ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -783,7 +819,7 @@ A website: http://example.org/ ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -854,7 +890,7 @@ func TestParseRepliesLinkRelationWithHTMLType(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -898,7 +934,7 @@ func TestParseRepliesLinkRelationWithXHTMLType(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -937,7 +973,7 @@ func TestParseRepliesLinkRelationWithNoType(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -977,7 +1013,7 @@ func TestAbsoluteCommentsURL(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } diff --git a/reader/atom/parser.go b/reader/atom/parser.go index 9a9cb578..e4e66819 100644 --- a/reader/atom/parser.go +++ b/reader/atom/parser.go @@ -15,11 +15,11 @@ import ( ) type atomFeed interface { - Transform() *model.Feed + Transform(baseURL string) *model.Feed } // Parse returns a normalized feed struct from a Atom feed. -func Parse(r io.Reader) (*model.Feed, *errors.LocalizedError) { +func Parse(baseURL string, r io.Reader) (*model.Feed, *errors.LocalizedError) { var buf bytes.Buffer tee := io.TeeReader(r, &buf) @@ -36,7 +36,7 @@ func Parse(r io.Reader) (*model.Feed, *errors.LocalizedError) { return nil, errors.NewLocalizedError("Unable to parse Atom feed: %q", err) } - return rawFeed.Transform(), nil + return rawFeed.Transform(baseURL), nil } func getAtomFeedVersion(data io.Reader) string { diff --git a/reader/feed/handler.go b/reader/feed/handler.go index 96609544..1fe5a0d5 100644 --- a/reader/feed/handler.go +++ b/reader/feed/handler.go @@ -58,7 +58,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, return nil, errors.NewLocalizedError(errDuplicate, response.EffectiveURL) } - subscription, parseErr := parser.ParseFeed(response.BodyAsString()) + subscription, parseErr := parser.ParseFeed(response.EffectiveURL, response.BodyAsString()) if parseErr != nil { return nil, parseErr } @@ -137,7 +137,7 @@ func (h *Handler) RefreshFeed(userID, feedID int64) error { if originalFeed.IgnoreHTTPCache || response.IsModified(originalFeed.EtagHeader, originalFeed.LastModifiedHeader) { logger.Debug("[Handler:RefreshFeed] Feed #%d has been modified", feedID) - updatedFeed, parseErr := parser.ParseFeed(response.BodyAsString()) + updatedFeed, parseErr := parser.ParseFeed(response.EffectiveURL, response.BodyAsString()) if parseErr != nil { originalFeed.WithError(parseErr.Localize(printer)) h.store.UpdateFeedError(originalFeed) diff --git a/reader/json/json.go b/reader/json/json.go index 5eca5e34..45e28888 100644 --- a/reader/json/json.go +++ b/reader/json/json.go @@ -55,12 +55,22 @@ func (j *jsonFeed) GetAuthor() string { return getAuthor(j.Author) } -func (j *jsonFeed) Transform() *model.Feed { - feed := new(model.Feed) - feed.FeedURL = j.FeedURL - feed.SiteURL = j.SiteURL - feed.Title = strings.TrimSpace(j.Title) +func (j *jsonFeed) Transform(baseURL string) *model.Feed { + var err error + feed := new(model.Feed) + + feed.FeedURL, err = url.AbsoluteURL(baseURL, j.FeedURL) + if err != nil { + feed.FeedURL = j.FeedURL + } + + feed.SiteURL, err = url.AbsoluteURL(baseURL, j.SiteURL) + if err != nil { + feed.SiteURL = j.SiteURL + } + + feed.Title = strings.TrimSpace(j.Title) if feed.Title == "" { feed.Title = feed.SiteURL } diff --git a/reader/json/parser.go b/reader/json/parser.go index babbde15..cb8638bf 100644 --- a/reader/json/parser.go +++ b/reader/json/parser.go @@ -13,12 +13,12 @@ import ( ) // Parse returns a normalized feed struct from a JON feed. -func Parse(data io.Reader) (*model.Feed, *errors.LocalizedError) { +func Parse(baseURL string, data io.Reader) (*model.Feed, *errors.LocalizedError) { feed := new(jsonFeed) decoder := json.NewDecoder(data) if err := decoder.Decode(&feed); err != nil { return nil, errors.NewLocalizedError("Unable to parse JSON Feed: %q", err) } - return feed.Transform(), nil + return feed.Transform(baseURL), nil } diff --git a/reader/json/parser_test.go b/reader/json/parser_test.go index 191423a0..93d8189a 100644 --- a/reader/json/parser_test.go +++ b/reader/json/parser_test.go @@ -31,7 +31,7 @@ func TestParseJsonFeed(t *testing.T) { ] }` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -113,7 +113,7 @@ func TestParsePodcast(t *testing.T) { ] }` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://therecord.co/feed.json", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -197,7 +197,7 @@ func TestParseEntryWithoutAttachmentURL(t *testing.T) { ] }` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://therecord.co/feed.json", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -226,7 +226,7 @@ func TestParseFeedWithRelativeURL(t *testing.T) { ] }` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -258,7 +258,7 @@ func TestParseAuthor(t *testing.T) { ] }` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -287,7 +287,7 @@ func TestParseFeedWithoutTitle(t *testing.T) { ] }` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -313,7 +313,7 @@ func TestParseFeedItemWithInvalidDate(t *testing.T) { ] }` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -341,7 +341,7 @@ func TestParseFeedItemWithoutID(t *testing.T) { ] }` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -368,7 +368,7 @@ func TestParseFeedItemWithoutTitle(t *testing.T) { ] }` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -395,7 +395,7 @@ func TestParseTruncateItemTitle(t *testing.T) { ] }` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -411,7 +411,7 @@ func TestParseTruncateItemTitle(t *testing.T) { func TestParseInvalidJSON(t *testing.T) { data := `garbage` - _, err := Parse(bytes.NewBufferString(data)) + _, err := Parse("https://example.org/feed.json", bytes.NewBufferString(data)) if err == nil { t.Error("Parse should returns an error") } diff --git a/reader/parser/parser.go b/reader/parser/parser.go index 726f3552..6214fdb8 100644 --- a/reader/parser/parser.go +++ b/reader/parser/parser.go @@ -16,16 +16,16 @@ import ( ) // ParseFeed analyzes the input data and returns a normalized feed object. -func ParseFeed(data string) (*model.Feed, *errors.LocalizedError) { +func ParseFeed(baseURL, data string) (*model.Feed, *errors.LocalizedError) { switch DetectFeedFormat(data) { case FormatAtom: - return atom.Parse(strings.NewReader(data)) + return atom.Parse(baseURL, strings.NewReader(data)) case FormatRSS: - return rss.Parse(strings.NewReader(data)) + return rss.Parse(baseURL, strings.NewReader(data)) case FormatJSON: - return json.Parse(strings.NewReader(data)) + return json.Parse(baseURL, strings.NewReader(data)) case FormatRDF: - return rdf.Parse(strings.NewReader(data)) + return rdf.Parse(baseURL, strings.NewReader(data)) default: return nil, errors.NewLocalizedError("Unsupported feed format") } diff --git a/reader/parser/parser_test.go b/reader/parser/parser_test.go index ddbaceca..c9a4c019 100644 --- a/reader/parser/parser_test.go +++ b/reader/parser/parser_test.go @@ -34,7 +34,7 @@ func TestParseAtom(t *testing.T) { ` - feed, err := ParseFeed(data) + feed, err := ParseFeed("https://example.org/", data) if err != nil { t.Error(err) } @@ -44,6 +44,42 @@ func TestParseAtom(t *testing.T) { } } +func TestParseAtomFeedWithRelativeURL(t *testing.T) { + data := ` + + Example Feed + + + + + Test + + + /blog/article.html + 2003-12-13T18:30:02Z + Some text. + + + ` + + feed, err := ParseFeed("https://example.org/blog/atom.xml", data) + if err != nil { + t.Fatal(err) + } + + if feed.FeedURL != "https://example.org/blog/atom.xml" { + t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL) + } + + if feed.SiteURL != "https://example.org/blog" { + t.Errorf("Incorrect site URL, got: %s", feed.SiteURL) + } + + if feed.Entries[0].URL != "https://example.org/blog/article.html" { + t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL) + } +} + func TestParseRSS(t *testing.T) { data := ` @@ -60,7 +96,7 @@ func TestParseRSS(t *testing.T) { ` - feed, err := ParseFeed(data) + feed, err := ParseFeed("http://liftoff.msfc.nasa.gov/", data) if err != nil { t.Error(err) } @@ -70,6 +106,44 @@ func TestParseRSS(t *testing.T) { } } +func TestParseRSSFeedWithRelativeURL(t *testing.T) { + data := ` + + + Example Feed + /blog + + Example Entry + /blog/article.html + Something + Tue, 03 Jun 2003 09:39:21 GMT + 1234 + + + ` + + feed, err := ParseFeed("http://example.org/rss.xml", data) + if err != nil { + t.Error(err) + } + + if feed.Title != "Example Feed" { + t.Errorf("Incorrect title, got: %s", feed.Title) + } + + if feed.FeedURL != "http://example.org/rss.xml" { + t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL) + } + + if feed.SiteURL != "http://example.org/blog" { + t.Errorf("Incorrect site URL, got: %s", feed.SiteURL) + } + + if feed.Entries[0].URL != "http://example.org/blog/article.html" { + t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL) + } +} + func TestParseRDF(t *testing.T) { data := ` ` - feed, err := ParseFeed(data) + feed, err := ParseFeed("http://example.org/", data) if err != nil { t.Error(err) } @@ -99,6 +173,43 @@ func TestParseRDF(t *testing.T) { } } +func TestParseRDFWithRelativeURL(t *testing.T) { + data := ` + + + + RDF Example + /blog + + + + Title + /blog/article.html + Test + + ` + + feed, err := ParseFeed("http://example.org/rdf.xml", data) + if err != nil { + t.Error(err) + } + + if feed.FeedURL != "http://example.org/rdf.xml" { + t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL) + } + + if feed.SiteURL != "http://example.org/blog" { + t.Errorf("Incorrect site URL, got: %s", feed.SiteURL) + } + + if feed.Entries[0].URL != "http://example.org/blog/article.html" { + t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL) + } +} + func TestParseJson(t *testing.T) { data := `{ "version": "https://jsonfeed.org/version/1", @@ -119,7 +230,7 @@ func TestParseJson(t *testing.T) { ] }` - feed, err := ParseFeed(data) + feed, err := ParseFeed("https://example.org/feed.json", data) if err != nil { t.Error(err) } @@ -129,6 +240,43 @@ func TestParseJson(t *testing.T) { } } +func TestParseJsonFeedWithRelativeURL(t *testing.T) { + data := `{ + "version": "https://jsonfeed.org/version/1", + "title": "My Example Feed", + "home_page_url": "/blog", + "feed_url": "/blog/feed.json", + "items": [ + { + "id": "2", + "content_text": "This is a second item.", + "url": "/blog/article.html" + } + ] + }` + + feed, err := ParseFeed("https://example.org/blog/feed.json", data) + if err != nil { + t.Error(err) + } + + if feed.Title != "My Example Feed" { + t.Errorf("Incorrect title, got: %s", feed.Title) + } + + if feed.FeedURL != "https://example.org/blog/feed.json" { + t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL) + } + + if feed.SiteURL != "https://example.org/blog" { + t.Errorf("Incorrect site URL, got: %s", feed.SiteURL) + } + + if feed.Entries[0].URL != "https://example.org/blog/article.html" { + t.Errorf("Incorrect entry URL, got: %s", feed.Entries[0].URL) + } +} + func TestParseUnknownFeed(t *testing.T) { data := ` @@ -142,14 +290,14 @@ func TestParseUnknownFeed(t *testing.T) { ` - _, err := ParseFeed(data) + _, err := ParseFeed("https://example.org/", data) if err == nil { t.Error("ParseFeed must returns an error") } } func TestParseEmptyFeed(t *testing.T) { - _, err := ParseFeed("") + _, err := ParseFeed("", "") if err == nil { t.Error("ParseFeed must returns an error") } @@ -191,7 +339,7 @@ func TestDifferentEncodingWithResponse(t *testing.T) { t.Fatalf(`Encoding error for %q: %v`, tc.filename, encodingErr) } - feed, parseErr := ParseFeed(r.BodyAsString()) + feed, parseErr := ParseFeed("https://example.org/", r.BodyAsString()) if parseErr != nil { t.Fatalf(`Parsing error for %q - %q: %v`, tc.filename, tc.contentType, parseErr) } diff --git a/reader/rdf/parser.go b/reader/rdf/parser.go index 57a8e522..70a9a9e6 100644 --- a/reader/rdf/parser.go +++ b/reader/rdf/parser.go @@ -13,7 +13,7 @@ import ( ) // Parse returns a normalized feed struct from a RDF feed. -func Parse(data io.Reader) (*model.Feed, *errors.LocalizedError) { +func Parse(baseURL string, data io.Reader) (*model.Feed, *errors.LocalizedError) { feed := new(rdfFeed) decoder := xml.NewDecoder(data) err := decoder.Decode(feed) @@ -21,5 +21,5 @@ func Parse(data io.Reader) (*model.Feed, *errors.LocalizedError) { return nil, errors.NewLocalizedError("Unable to parse RDF feed: %q", err) } - return feed.Transform(), nil + return feed.Transform(baseURL), nil } diff --git a/reader/rdf/parser_test.go b/reader/rdf/parser_test.go index 0958f3ce..9383fb03 100644 --- a/reader/rdf/parser_test.go +++ b/reader/rdf/parser_test.go @@ -76,7 +76,7 @@ func TestParseRDFSample(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://xml.com/pub/rdf.xml", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -85,7 +85,7 @@ func TestParseRDFSample(t *testing.T) { t.Errorf("Incorrect title, got: %s", feed.Title) } - if feed.FeedURL != "" { + if feed.FeedURL != "http://xml.com/pub/rdf.xml" { t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL) } @@ -187,7 +187,7 @@ func TestParseRDFSampleWithDublinCore(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://meerkat.oreillynet.com/feed.rdf", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -196,7 +196,7 @@ func TestParseRDFSampleWithDublinCore(t *testing.T) { t.Errorf("Incorrect title, got: %s", feed.Title) } - if feed.FeedURL != "" { + if feed.FeedURL != "http://meerkat.oreillynet.com/feed.rdf" { t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL) } @@ -254,7 +254,7 @@ func TestParseItemWithOnlyFeedAuthor(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://meerkat.oreillynet.com", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -279,7 +279,7 @@ func TestParseItemRelativeURL(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://meerkat.oreillynet.com", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -308,7 +308,7 @@ func TestParseItemWithoutLink(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://meerkat.oreillynet.com", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -339,7 +339,7 @@ func TestParseItemWithDublicCoreDate(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://example.org", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -365,7 +365,7 @@ func TestParseItemWithoutDate(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://example.org", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -379,7 +379,7 @@ func TestParseItemWithoutDate(t *testing.T) { func TestParseInvalidXml(t *testing.T) { data := `garbage` - _, err := Parse(bytes.NewBufferString(data)) + _, err := Parse("http://example.org", bytes.NewBufferString(data)) if err == nil { t.Fatal("Parse should returns an error") } @@ -394,7 +394,7 @@ func TestParseFeedWithHTMLEntity(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://example.org", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -413,7 +413,7 @@ func TestParseFeedWithInvalidCharacterEntity(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://example.org", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -469,7 +469,7 @@ func TestParseFeedWithURLWrappedInSpaces(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://biorxiv.org", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -504,7 +504,7 @@ func TestParseRDFWithContentEncoded(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } diff --git a/reader/rdf/rdf.go b/reader/rdf/rdf.go index 73c3801b..337df206 100644 --- a/reader/rdf/rdf.go +++ b/reader/rdf/rdf.go @@ -25,10 +25,15 @@ type rdfFeed struct { DublinCoreFeedElement } -func (r *rdfFeed) Transform() *model.Feed { +func (r *rdfFeed) Transform(baseURL string) *model.Feed { + var err error feed := new(model.Feed) feed.Title = sanitizer.StripTags(r.Title) - feed.SiteURL = r.Link + feed.FeedURL = baseURL + feed.SiteURL, err = url.AbsoluteURL(baseURL, r.Link) + if err != nil { + feed.SiteURL = r.Link + } for _, item := range r.Items { entry := item.Transform() diff --git a/reader/rss/parser.go b/reader/rss/parser.go index 9ed773df..6c9af4ff 100644 --- a/reader/rss/parser.go +++ b/reader/rss/parser.go @@ -13,7 +13,7 @@ import ( ) // Parse returns a normalized feed struct from a RSS feed. -func Parse(data io.Reader) (*model.Feed, *errors.LocalizedError) { +func Parse(baseURL string, data io.Reader) (*model.Feed, *errors.LocalizedError) { feed := new(rssFeed) decoder := xml.NewDecoder(data) err := decoder.Decode(feed) @@ -21,5 +21,5 @@ func Parse(data io.Reader) (*model.Feed, *errors.LocalizedError) { return nil, errors.NewLocalizedError("Unable to parse RSS feed: %q", err) } - return feed.Transform(), nil + return feed.Transform(baseURL), nil } diff --git a/reader/rss/parser_test.go b/reader/rss/parser_test.go index 16dd1c2d..64d1e456 100644 --- a/reader/rss/parser_test.go +++ b/reader/rss/parser_test.go @@ -54,7 +54,7 @@ func TestParseRss2Sample(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("http://liftoff.msfc.nasa.gov/rss.xml", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -63,7 +63,7 @@ func TestParseRss2Sample(t *testing.T) { t.Errorf("Incorrect title, got: %s", feed.Title) } - if feed.FeedURL != "" { + if feed.FeedURL != "http://liftoff.msfc.nasa.gov/rss.xml" { t.Errorf("Incorrect feed URL, got: %s", feed.FeedURL) } @@ -105,7 +105,7 @@ func TestParseFeedWithoutTitle(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -126,7 +126,7 @@ func TestParseEntryWithoutTitle(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -149,7 +149,7 @@ func TestParseEntryWithMediaTitle(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -171,7 +171,7 @@ func TestParseEntryWithDCTitleOnly(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -192,7 +192,7 @@ func TestParseEntryWithoutLink(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -218,7 +218,7 @@ func TestParseEntryWithAtomLink(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -241,7 +241,7 @@ func TestParseEntryWithMultipleAtomLinks(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -261,7 +261,7 @@ func TestParseFeedURLWithAtomLink(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -289,7 +289,7 @@ func TestParseFeedWithWebmaster(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -316,7 +316,7 @@ func TestParseFeedWithManagingEditor(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -343,7 +343,7 @@ func TestParseEntryWithAuthorAndInnerHTML(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -375,7 +375,7 @@ func TestParseEntryWithNonStandardAtomAuthor(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -404,7 +404,7 @@ func TestParseEntryWithAtomAuthorEmail(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -433,7 +433,7 @@ func TestParseEntryWithAtomAuthor(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -459,7 +459,7 @@ func TestParseEntryWithDublinCoreAuthor(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -485,7 +485,7 @@ func TestParseEntryWithItunesAuthor(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -511,7 +511,7 @@ func TestParseFeedWithItunesAuthor(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -540,7 +540,7 @@ func TestParseFeedWithItunesOwner(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -568,7 +568,7 @@ func TestParseFeedWithItunesOwnerEmail(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -594,7 +594,7 @@ func TestParseEntryWithGooglePlayAuthor(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -620,7 +620,7 @@ func TestParseFeedWithGooglePlayAuthor(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -648,7 +648,7 @@ func TestParseEntryWithDublinCoreDate(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -676,7 +676,7 @@ func TestParseEntryWithContentEncoded(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -700,7 +700,7 @@ func TestParseEntryWithFeedBurnerLink(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -726,7 +726,7 @@ func TestParseEntryTitleWithWhitespaces(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -756,7 +756,7 @@ func TestParseEntryWithEnclosures(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -804,7 +804,7 @@ func TestParseEntryWithEmptyEnclosureURL(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -841,7 +841,7 @@ func TestParseEntryWithFeedBurnerEnclosures(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -882,7 +882,7 @@ func TestParseEntryWithRelativeURL(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -908,7 +908,7 @@ func TestParseEntryWithCommentsURL(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -933,7 +933,7 @@ func TestParseEntryWithInvalidCommentsURL(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -945,7 +945,7 @@ func TestParseEntryWithInvalidCommentsURL(t *testing.T) { func TestParseInvalidXml(t *testing.T) { data := `garbage` - _, err := Parse(bytes.NewBufferString(data)) + _, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err == nil { t.Error("Parse should returns an error") } @@ -960,7 +960,7 @@ func TestParseWithHTMLEntity(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -979,7 +979,7 @@ func TestParseWithInvalidCharacterEntity(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -1013,7 +1013,7 @@ func TestParseEntryWithMediaGroup(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -1071,7 +1071,7 @@ func TestParseEntryWithMediaContent(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -1122,7 +1122,7 @@ func TestParseEntryWithMediaPeerLink(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -1174,7 +1174,7 @@ func TestEntryDescriptionFromItunesSummary(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -1205,7 +1205,7 @@ func TestEntryDescriptionFromItunesSubtitle(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } @@ -1239,7 +1239,7 @@ func TestEntryDescriptionFromGooglePlayDescription(t *testing.T) { ` - feed, err := Parse(bytes.NewBufferString(data)) + feed, err := Parse("https://example.org/", bytes.NewBufferString(data)) if err != nil { t.Fatal(err) } diff --git a/reader/rss/rss.go b/reader/rss/rss.go index 490f9253..51d52ce6 100644 --- a/reader/rss/rss.go +++ b/reader/rss/rss.go @@ -35,12 +35,24 @@ type rssFeed struct { PodcastFeedElement } -func (r *rssFeed) Transform() *model.Feed { - feed := new(model.Feed) - feed.SiteURL = r.siteURL() - feed.FeedURL = r.feedURL() - feed.Title = strings.TrimSpace(r.Title) +func (r *rssFeed) Transform(baseURL string) *model.Feed { + var err error + feed := new(model.Feed) + + siteURL := r.siteURL() + feed.SiteURL, err = url.AbsoluteURL(baseURL, siteURL) + if err != nil { + feed.SiteURL = siteURL + } + + feedURL := r.feedURL() + feed.FeedURL, err = url.AbsoluteURL(baseURL, feedURL) + if err != nil { + feed.FeedURL = feedURL + } + + feed.Title = strings.TrimSpace(r.Title) if feed.Title == "" { feed.Title = feed.SiteURL } diff --git a/url/url_test.go b/url/url_test.go index ea488cf1..f7722881 100644 --- a/url/url_test.go +++ b/url/url_test.go @@ -23,13 +23,13 @@ func TestIsAbsoluteURL(t *testing.T) { func TestAbsoluteURL(t *testing.T) { scenarios := [][]string{ - []string{"https://example.org/path/file.ext", "https://example.org/folder/", "/path/file.ext"}, - []string{"https://example.org/folder/path/file.ext", "https://example.org/folder/", "path/file.ext"}, - []string{"https://example.org/path/file.ext", "https://example.org/folder", "path/file.ext"}, - []string{"https://example.org/path/file.ext", "https://example.org/folder/", "https://example.org/path/file.ext"}, - []string{"https://static.example.org/path/file.ext", "https://www.example.org/", "//static.example.org/path/file.ext"}, - []string{"magnet:?xt=urn:btih:c12fe1c06bba254a9dc9f519b335aa7c1367a88a", "https://www.example.org/", "magnet:?xt=urn:btih:c12fe1c06bba254a9dc9f519b335aa7c1367a88a"}, - []string{"magnet:?xt.1=urn:sha1:YNCKHTQCWBTRNJIV4WNAE52SJUQCZO5C&xt.2=urn:sha1:TXGCZQTH26NL6OUQAJJPFALHG2LTGBC7", "https://www.example.org/", "magnet:?xt.1=urn:sha1:YNCKHTQCWBTRNJIV4WNAE52SJUQCZO5C&xt.2=urn:sha1:TXGCZQTH26NL6OUQAJJPFALHG2LTGBC7"}, + {"https://example.org/path/file.ext", "https://example.org/folder/", "/path/file.ext"}, + {"https://example.org/folder/path/file.ext", "https://example.org/folder/", "path/file.ext"}, + {"https://example.org/path/file.ext", "https://example.org/folder", "path/file.ext"}, + {"https://example.org/path/file.ext", "https://example.org/folder/", "https://example.org/path/file.ext"}, + {"https://static.example.org/path/file.ext", "https://www.example.org/", "//static.example.org/path/file.ext"}, + {"magnet:?xt=urn:btih:c12fe1c06bba254a9dc9f519b335aa7c1367a88a", "https://www.example.org/", "magnet:?xt=urn:btih:c12fe1c06bba254a9dc9f519b335aa7c1367a88a"}, + {"magnet:?xt.1=urn:sha1:YNCKHTQCWBTRNJIV4WNAE52SJUQCZO5C&xt.2=urn:sha1:TXGCZQTH26NL6OUQAJJPFALHG2LTGBC7", "https://www.example.org/", "magnet:?xt.1=urn:sha1:YNCKHTQCWBTRNJIV4WNAE52SJUQCZO5C&xt.2=urn:sha1:TXGCZQTH26NL6OUQAJJPFALHG2LTGBC7"}, } for _, scenario := range scenarios {