Do not follow redirects when trying known feed URLs

Some websites redirects unknown URLs to the home page.
As result, the list of known URLs is returned to the subscription list.
We don't want the user to choose between invalid feed URLs.
This commit is contained in:
Frédéric Guillot 2020-11-06 17:39:10 -08:00
parent 40e983664c
commit 246a48359c
2 changed files with 26 additions and 3 deletions

View File

@ -52,7 +52,8 @@ type Client struct {
requestPassword string
requestUserAgent string
useProxy bool
useProxy bool
doNotFollowRedirects bool
ClientTimeout int
ClientMaxBodySize int64
@ -124,12 +125,18 @@ func (c *Client) WithCacheHeaders(etagHeader, lastModifiedHeader string) *Client
return c
}
// WithProxy enable proxy for the current HTTP request.
// WithProxy enables proxy for the current HTTP request.
func (c *Client) WithProxy() *Client {
c.useProxy = true
return c
}
// WithoutRedirects disables HTTP redirects.
func (c *Client) WithoutRedirects() *Client {
c.doNotFollowRedirects = true
return c
}
// WithUserAgent defines the User-Agent header to use for HTTP requests.
func (c *Client) WithUserAgent(userAgent string) *Client {
if userAgent != "" {
@ -266,7 +273,10 @@ func (c *Client) buildRequest(method string, body io.Reader) (*http.Request, err
}
func (c *Client) buildClient() http.Client {
client := http.Client{Timeout: time.Duration(c.ClientTimeout) * time.Second}
client := http.Client{
Timeout: time.Duration(c.ClientTimeout) * time.Second,
}
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
@ -284,6 +294,12 @@ func (c *Client) buildClient() http.Client {
IdleConnTimeout: 10 * time.Second,
}
if c.doNotFollowRedirects {
client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
}
}
if c.useProxy && c.ClientProxyURL != "" {
proxyURL, err := url.Parse(c.ClientProxyURL)
if err != nil {

View File

@ -156,9 +156,16 @@ func tryWellKnownUrls(websiteURL, userAgent, username, password string) (Subscri
if err != nil {
continue
}
clt := client.NewClientWithConfig(fullURL, config.Opts)
clt.WithCredentials(username, password)
clt.WithUserAgent(userAgent)
// Some websites redirects unknown URLs to the home page.
// As result, the list of known URLs is returned to the subscription list.
// We don't want the user to choose between invalid feed URLs.
clt.WithoutRedirects()
response, err := clt.Get()
if err != nil {
continue