diff --git a/reader/scraper/scraper.go b/reader/scraper/scraper.go index dbf5d2eb..9044a1cb 100644 --- a/reader/scraper/scraper.go +++ b/reader/scraper/scraper.go @@ -118,11 +118,17 @@ func followTheOnlyLink(websiteURL, content string, rules, userAgent string, cook if err != nil { return "", err } + + if len(document.Find("body").Nodes) == 0 { + return content, nil + } + body := document.Find("body").Nodes[0] if body.FirstChild.NextSibling != nil || body.FirstChild.Data != "a" { return content, nil } + // the body has only one child of var href string for _, attr := range body.FirstChild.Attr { @@ -131,16 +137,20 @@ func followTheOnlyLink(websiteURL, content string, rules, userAgent string, cook break } } + if href == "" { return content, nil } + href, err = url.AbsoluteURL(websiteURL, href) if err != nil { return "", err } + sameSite := url.Domain(websiteURL) == url.Domain(href) if sameSite { return fetchURL(href, rules, userAgent, cookie, allowSelfSignedCertificates, useProxy) } + return fetchURL(href, rules, userAgent, "", false, false) }