rdelaage
/
miniflux-v2
mirror of https://github.com/miniflux/v2.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

add proxy arg in scraper.Fetch

This commit is contained in:
hulb 2021-08-28 17:30:04 +08:00 committed by fguillot
parent 2f6895e118
commit 01f678c3b1
2 changed files with 6 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
reader/processor/processor.go
View File

@ -56,6 +56,7 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed) {
feed.UserAgent,
feed.Cookie,
feed.AllowSelfSignedCertificates,
feed.FetchViaProxy,
)
if config.Opts.HasMetricsCollector() {
@ -118,6 +119,7 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry) error {
entry.Feed.UserAgent,
entry.Feed.Cookie,
feed.AllowSelfSignedCertificates,
feed.FetchViaProxy,
)
if config.Opts.HasMetricsCollector() {

5
reader/scraper/scraper.go
View File

@ -20,10 +20,13 @@ import (
)
// Fetch downloads a web page and returns relevant contents.
func Fetch(websiteURL, rules, userAgent string, cookie string, allowSelfSignedCertificates bool) (string, error) {
func Fetch(websiteURL, rules, userAgent string, cookie string, allowSelfSignedCertificates, useProxy bool) (string, error) {
clt := client.NewClientWithConfig(websiteURL, config.Opts)
clt.WithUserAgent(userAgent)
clt.WithCookie(cookie)
if useProxy {
clt.WithProxy()
}
clt.AllowSelfSignedCertificates = allowSelfSignedCertificates
response, err := clt.Get()