mirror of https://github.com/miniflux/v2.git
add proxy arg in scraper.Fetch
This commit is contained in:
parent
2f6895e118
commit
01f678c3b1
|
@ -56,6 +56,7 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed) {
|
||||||
feed.UserAgent,
|
feed.UserAgent,
|
||||||
feed.Cookie,
|
feed.Cookie,
|
||||||
feed.AllowSelfSignedCertificates,
|
feed.AllowSelfSignedCertificates,
|
||||||
|
feed.FetchViaProxy,
|
||||||
)
|
)
|
||||||
|
|
||||||
if config.Opts.HasMetricsCollector() {
|
if config.Opts.HasMetricsCollector() {
|
||||||
|
@ -118,6 +119,7 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry) error {
|
||||||
entry.Feed.UserAgent,
|
entry.Feed.UserAgent,
|
||||||
entry.Feed.Cookie,
|
entry.Feed.Cookie,
|
||||||
feed.AllowSelfSignedCertificates,
|
feed.AllowSelfSignedCertificates,
|
||||||
|
feed.FetchViaProxy,
|
||||||
)
|
)
|
||||||
|
|
||||||
if config.Opts.HasMetricsCollector() {
|
if config.Opts.HasMetricsCollector() {
|
||||||
|
|
|
@ -20,10 +20,13 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
// Fetch downloads a web page and returns relevant contents.
|
// Fetch downloads a web page and returns relevant contents.
|
||||||
func Fetch(websiteURL, rules, userAgent string, cookie string, allowSelfSignedCertificates bool) (string, error) {
|
func Fetch(websiteURL, rules, userAgent string, cookie string, allowSelfSignedCertificates, useProxy bool) (string, error) {
|
||||||
clt := client.NewClientWithConfig(websiteURL, config.Opts)
|
clt := client.NewClientWithConfig(websiteURL, config.Opts)
|
||||||
clt.WithUserAgent(userAgent)
|
clt.WithUserAgent(userAgent)
|
||||||
clt.WithCookie(cookie)
|
clt.WithCookie(cookie)
|
||||||
|
if useProxy {
|
||||||
|
clt.WithProxy()
|
||||||
|
}
|
||||||
clt.AllowSelfSignedCertificates = allowSelfSignedCertificates
|
clt.AllowSelfSignedCertificates = allowSelfSignedCertificates
|
||||||
|
|
||||||
response, err := clt.Get()
|
response, err := clt.Get()
|
||||||
|
|
Loading…
Reference in New Issue