rdelaage
/
miniflux-v2
mirror of https://github.com/miniflux/v2.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add HTTP proxy option for subscriptions

This commit is contained in:
Kebin Liu 2020-09-10 14:28:54 +08:00 committed by GitHub
parent 0f258fd55b
commit cf7712acea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 201 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
api/feed.go
View File

@ -51,6 +51,7 @@ func (h *handler) createFeed(w http.ResponseWriter, r *http.Request) {
feedInfo.Password,
feedInfo.ScraperRules,
feedInfo.RewriteRules,
feedInfo.FetchViaProxy,
)
if err != nil {
json.ServerError(w, r, err)

26
api/payload.go
View File

@ -24,21 +24,23 @@ type entriesResponse struct {
}
type feedCreation struct {
FeedURL string `json:"feed_url"`
CategoryID int64 `json:"category_id"`
UserAgent string `json:"user_agent"`
Username string `json:"username"`
Password string `json:"password"`
Crawler bool `json:"crawler"`
ScraperRules string `json:"scraper_rules"`
RewriteRules string `json:"rewrite_rules"`
FeedURL string `json:"feed_url"`
CategoryID int64 `json:"category_id"`
UserAgent string `json:"user_agent"`
Username string `json:"username"`
Password string `json:"password"`
Crawler bool `json:"crawler"`
FetchViaProxy bool `json:"fetch_via_proxy"`
ScraperRules string `json:"scraper_rules"`
RewriteRules string `json:"rewrite_rules"`
}
type subscriptionDiscovery struct {
URL string `json:"url"`
UserAgent string `json:"user_agent"`
Username string `json:"username"`
Password string `json:"password"`
URL string `json:"url"`
UserAgent string `json:"user_agent"`
Username string `json:"username"`
Password string `json:"password"`
FetchViaProxy bool `json:"fetch_via_proxy"`
}
type feedModification struct {

1
api/subscription.go
View File

@ -23,6 +23,7 @@ func (h *handler) getSubscriptions(w http.ResponseWriter, r *http.Request) {
subscriptionInfo.UserAgent,
subscriptionInfo.Username,
subscriptionInfo.Password,
subscriptionInfo.FetchViaProxy,
)
if finderErr != nil {
json.ServerError(w, r, finderErr)

14
config/options.go
View File

@ -50,6 +50,7 @@ const (
defaultPocketConsumerKey = ""
defaultHTTPClientTimeout = 20
defaultHTTPClientMaxBodySize = 15
defaultHTTPClientProxy = ""
defaultAuthProxyHeader = ""
defaultAuthProxyUserCreation = false
)
@ -96,6 +97,7 @@ type Options struct {
pocketConsumerKey string
httpClientTimeout int
httpClientMaxBodySize int64
httpClientProxy string
authProxyHeader string
authProxyUserCreation bool
}
@ -141,6 +143,7 @@ func NewOptions() *Options {
pocketConsumerKey: defaultPocketConsumerKey,
httpClientTimeout: defaultHTTPClientTimeout,
httpClientMaxBodySize: defaultHTTPClientMaxBodySize * 1024 * 1024,
httpClientProxy: defaultHTTPClientProxy,
authProxyHeader: defaultAuthProxyHeader,
authProxyUserCreation: defaultAuthProxyUserCreation,
}
@ -349,6 +352,16 @@ func (o *Options) HTTPClientMaxBodySize() int64 {
return o.httpClientMaxBodySize
}
// HTTPClientProxy returns the proxy URL for HTTP client.
func (o *Options) HTTPClientProxy() string {
return o.httpClientProxy
}
// HasHTTPClientProxyConfigured returns true if the HTTP proxy is configured.
func (o *Options) HasHTTPClientProxyConfigured() bool {
return o.httpClientProxy != ""
}
// AuthProxyHeader returns an HTTP header name that contains username for
// authentication using auth proxy.
func (o *Options) AuthProxyHeader() string {
@ -403,6 +416,7 @@ func (o *Options) String() string {
builder.WriteString(fmt.Sprintf("OAUTH2_PROVIDER: %v\n", o.oauth2Provider))
builder.WriteString(fmt.Sprintf("HTTP_CLIENT_TIMEOUT: %v\n", o.httpClientTimeout))
builder.WriteString(fmt.Sprintf("HTTP_CLIENT_MAX_BODY_SIZE: %v\n", o.httpClientMaxBodySize))
builder.WriteString(fmt.Sprintf("HTTP_CLIENT_PROXY: %v\n", o.httpClientProxy))
builder.WriteString(fmt.Sprintf("AUTH_PROXY_HEADER: %v\n", o.authProxyHeader))
builder.WriteString(fmt.Sprintf("AUTH_PROXY_USER_CREATION: %v\n", o.authProxyUserCreation))
return builder.String()

2
config/parser.go
View File

@ -184,6 +184,8 @@ func (p *Parser) parseLines(lines []string) (err error) {
p.opts.httpClientTimeout = parseInt(value, defaultHTTPClientTimeout)
case "HTTP_CLIENT_MAX_BODY_SIZE":
p.opts.httpClientMaxBodySize = int64(parseInt(value, defaultHTTPClientMaxBodySize) * 1024 * 1024)
case "HTTP_CLIENT_PROXY":
p.opts.httpClientProxy = parseString(value, defaultHTTPClientProxy)
case "AUTH_PROXY_HEADER":
p.opts.authProxyHeader = parseString(value, defaultAuthProxyHeader)
case "AUTH_PROXY_USER_CREATION":

2
database/migration.go
View File

@ -12,7 +12,7 @@ import (
"miniflux.app/logger"
)
const schemaVersion = 34
const schemaVersion = 35
// Migrate executes database migrations.
func Migrate(db *sql.DB) {

3
database/sql.go
View File

@ -187,6 +187,8 @@ create index entries_user_feed_idx on entries (user_id, feed_id);
`,
"schema_version_33": `alter table users add column show_reading_time boolean default 't';`,
"schema_version_34": `CREATE INDEX entries_id_user_status_idx ON entries USING btree (id, user_id, status);`,
"schema_version_35": `alter table feeds add column fetch_via_proxy bool default false;
`,
"schema_version_4": `create type entry_sorting_direction as enum('asc', 'desc');
alter table users add column entry_direction entry_sorting_direction default 'asc';
`,
@ -244,6 +246,7 @@ var SqlMapChecksums = map[string]string{
"schema_version_32": "5b4de8dd2d7e3c6ae4150e0e3931df2ee989f2c667145bd67294e5a5f3fae456",
"schema_version_33": "bf38514efeb6c12511f41b1cc484f92722240b0a6ae874c32a958dfea3433d02",
"schema_version_34": "1a3e036f652fc98b7564a27013f04e1eb36dd0d68893c723168f134dc1065822",
"schema_version_35": "162a55df78eed4b9c9c141878132d5f1d97944b96f35a79e38f55716cdd6b3d2",
"schema_version_4": "216ea3a7d3e1704e40c797b5dc47456517c27dbb6ca98bf88812f4f63d74b5d9",
"schema_version_5": "46397e2f5f2c82116786127e9f6a403e975b14d2ca7b652a48cd1ba843e6a27c",
"schema_version_6": "9d05b4fb223f0e60efc716add5048b0ca9c37511cf2041721e20505d6d798ce4",

1
database/sql/schema_version_35.sql Normal file
View File

@ -0,0 +1 @@
alter table feeds add column fetch_via_proxy bool default false;

22
http/client/client.go
View File

@ -47,6 +47,7 @@ type Client struct {
password string
userAgent string
Insecure bool
fetchViaProxy bool
}
func (c *Client) String() string {
@ -93,6 +94,12 @@ func (c *Client) WithCacheHeaders(etagHeader, lastModifiedHeader string) *Client
return c
}
// WithProxy enable proxy for current HTTP client request.
func (c *Client) WithProxy() *Client {
c.fetchViaProxy = true
return c
}
// WithUserAgent defines the User-Agent header to use for outgoing requests.
func (c *Client) WithUserAgent(userAgent string) *Client {
if userAgent != "" {
@ -230,12 +237,23 @@ func (c *Client) buildRequest(method string, body io.Reader) (*http.Request, err
func (c *Client) buildClient() http.Client {
client := http.Client{Timeout: time.Duration(config.Opts.HTTPClientTimeout()) * time.Second}
transport := &http.Transport{}
if c.Insecure {
client.Transport = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
}
if c.fetchViaProxy && config.Opts.HasHTTPClientProxyConfigured() {
proxyURL, err := url.Parse(config.Opts.HTTPClientProxy())
if err != nil {
logger.Error("[HttpClient] Proxy URL error: %v", err)
} else {
logger.Debug("[HttpClient] Use proxy: %s", proxyURL)
transport.Proxy = http.ProxyURL(proxyURL)
}
}
client.Transport = transport
return client
}

33
locale/translations.go
View File

@ -254,6 +254,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Extraktionsregeln",
"form.feed.label.rewrite_rules": "Umschreiberegeln",
"form.feed.label.ignore_http_cache": "Ignoriere HTTP-cache",
"form.feed.label.fetch_via_proxy": "Über Proxy abrufen",
"form.feed.label.disabled": "Dieses Abonnement nicht aktualisieren",
"form.category.label.title": "Titel",
"form.user.label.username": "Benutzername",
@ -599,6 +600,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Scraper Rules",
"form.feed.label.rewrite_rules": "Rewrite Rules",
"form.feed.label.ignore_http_cache": "Ignore HTTP cache",
"form.feed.label.fetch_via_proxy": "Fetch via proxy",
"form.feed.label.disabled": "Do not refresh this feed",
"form.category.label.title": "Title",
"form.user.label.username": "Username",
@ -924,6 +926,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Reglas de raspador",
"form.feed.label.rewrite_rules": "Reglas de reescribir",
"form.feed.label.ignore_http_cache": "Ignorar caché HTTP",
"form.feed.label.fetch_via_proxy": "Buscar a través de proxy",
"form.feed.label.disabled": "No actualice este feed",
"form.category.label.title": "Título",
"form.user.label.username": "Nombre de usuario",
@ -1249,6 +1252,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Règles pour récupérer le contenu original",
"form.feed.label.rewrite_rules": "Règles de réécriture",
"form.feed.label.ignore_http_cache": "Ignore cache HTTP",
"form.feed.label.fetch_via_proxy": "Récupérer via proxy",
"form.feed.label.disabled": "Ne pas actualiser ce flux",
"form.category.label.title": "Titre",
"form.user.label.username": "Nom d'utilisateur",
@ -1594,6 +1598,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Regole di estrazione del contenuto",
"form.feed.label.rewrite_rules": "Regole di impaginazione del contenuto",
"form.feed.label.ignore_http_cache": "Ignora cache HTTP",
"form.feed.label.fetch_via_proxy": "Recuperare tramite proxy",
"form.feed.label.disabled": "Non aggiornare questo feed",
"form.category.label.title": "Titolo",
"form.user.label.username": "Nome utente",
@ -1919,6 +1924,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "スクラップルール",
"form.feed.label.rewrite_rules": "Rewrite ルール",
"form.feed.label.ignore_http_cache": "HTTPキャッシュを無視",
"form.feed.label.fetch_via_proxy": "プロキシ経由でフェッチ",
"form.feed.label.disabled": "このフィードを更新しない",
"form.category.label.title": "タイトル",
"form.user.label.username": "ユーザー名",
@ -2244,6 +2250,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Scraper regels",
"form.feed.label.rewrite_rules": "Rewrite regels",
"form.feed.label.ignore_http_cache": "Negeer HTTP-cache",
"form.feed.label.fetch_via_proxy": "Ophalen via proxy",
"form.feed.label.disabled": "Vernieuw deze feed niet",
"form.category.label.title": "Naam",
"form.user.label.username": "Gebruikersnaam",
@ -2589,6 +2596,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Zasady ekstrakcji",
"form.feed.label.rewrite_rules": "Reguły zapisu",
"form.feed.label.ignore_http_cache": "Zignoruj ​​pamięć podręczną HTTP",
"form.feed.label.fetch_via_proxy": "Pobierz przez proxy",
"form.feed.label.disabled": "Не обновлять этот канал",
"form.category.label.title": "Tytuł",
"form.user.label.username": "Nazwa użytkownika",
@ -2939,6 +2947,7 @@ var translations = map[string]string{
"form.feed.label.rewrite_rules": "Regras para o Rewrite",
"form.feed.label.ignore_http_cache": "Ignorar cache HTTP",
"form.feed.label.disabled": "Não atualizar esta fonte",
"form.feed.label.fetch_via_proxy": "Buscar via proxy",
"form.category.label.title": "Título",
"form.user.label.username": "Nome de usuário",
"form.user.label.password": "Senha",
@ -3265,6 +3274,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Правила Scraper",
"form.feed.label.rewrite_rules": "Правила Rewrite",
"form.feed.label.ignore_http_cache": "Игнорировать HTTP-кеш",
"form.feed.label.fetch_via_proxy": "Получить через прокси",
"form.feed.label.disabled": "Не обновлять этот канал",
"form.category.label.title": "Название",
"form.user.label.username": "Имя пользователя",
@ -3594,6 +3604,7 @@ var translations = map[string]string{
"form.feed.label.scraper_rules": "Scraper 规则",
"form.feed.label.rewrite_rules": "重写规则",
"form.feed.label.ignore_http_cache": "忽略HTTP缓存",
"form.feed.label.fetch_via_proxy": "通过代理获取",
"form.feed.label.disabled": "请勿刷新此Feed",
"form.category.label.title": "标题",
"form.user.label.username": "用户名",
@ -3684,15 +3695,15 @@ var translations = map[string]string{
}
var translationsChecksums = map[string]string{
"de_DE": "21e1bfb0f43d71efe38812b4337ddf6980c11ed18f4d06446ff7eda9dfa6b1f1",
"en_US": "30cbcb2170782f1e66f69066947bf053f68065d7b270eea879f2c573819dd52b",
"es_ES": "50dc7c8c2db7368bae133f5b455721470d314321153d41e4f27436a0f3f176e6",
"fr_FR": "373fd2db868961758bd1483c34f117b03aadea17080f268bc8bbd0acdfbc5eed",
"it_IT": "8d8f0bd75b4e7dec9370647c888dd9438b691130d9c41f839cdfff8cbc606cb5",
"ja_JP": "ec3a21c547e4625ad359624e43ba31b556fb8d8b8ff7fc7a20df089317db99b3",
"nl_NL": "20e180be2375f07ec02eb05f372a9102c13037a79e5651ce9bd41507fd2180d2",
"pl_PL": "b1526955641823708b4c1ca753b61e1e0561d0a3d33da3f62170540903031b0d",
"pt_BR": "cf8e131d39daac82d3157c6538c0643392a06358b7bc98be8579412ebd63f60e",
"ru_RU": "4056e4e94861835d44064273371adbbded7190e2b719769886eb99e6c9feaf82",
"zh_CN": "044abb0a34eee3d8d5597811d40166762311b8e4cd08b891796113790cc775f0",
"de_DE": "8f96cb46f5a7e8f64ee8f10176dc3a2f3d53953d250317da83a79d0700b47c82",
"en_US": "d33324caed406ecf6ce03920b15e235d46b258457a8bd48cd1ade685b9a3ad6b",
"es_ES": "2ff9333218dba2b86cb84f377dad66b9dc73848aee6bb09889cbdc10e58ca077",
"fr_FR": "07dc2cfdbc14cdf16312423158656f5526d3c3c7be490abf5503109a408e5056",
"it_IT": "d4f68a507e1deb9fab3aa38fb78d9e9e4040386d6f36611ec5f105adfb4b0d03",
"ja_JP": "5c4c063ebaee14bed941b020e0d19de5ef5e8d3bf11c1967b1f321d57d5af6a9",
"nl_NL": "f862027e192be7a09730470acc2639971c4abf01b256d5bb81246960cc54adcf",
"pl_PL": "35147e55f1800964d268dc04b9cc25a9c8fa98077f759c5d3a5bd339f0eee53e",
"pt_BR": "2461105ebc2a2d57b3a63a29ee21f74e3d1eba54c049abcfd077dd30acc8d0a2",
"ru_RU": "402f15d3c68e008a1ffa3dddb4002126a29c1cf93359a64bca944a8da541da72",
"zh_CN": "bbea61a08bec37518d8c8c7735c7b8001d011a1d43ddb15ab639fee11b45ca87",
}

1
locale/translations/de_DE.json
View File

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "Extraktionsregeln",
"form.feed.label.rewrite_rules": "Umschreiberegeln",
"form.feed.label.ignore_http_cache": "Ignoriere HTTP-cache",
"form.feed.label.fetch_via_proxy": "Über Proxy abrufen",
"form.feed.label.disabled": "Dieses Abonnement nicht aktualisieren",
"form.category.label.title": "Titel",
"form.user.label.username": "Benutzername",

1
locale/translations/en_US.json
View File

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "Scraper Rules",
"form.feed.label.rewrite_rules": "Rewrite Rules",
"form.feed.label.ignore_http_cache": "Ignore HTTP cache",
"form.feed.label.fetch_via_proxy": "Fetch via proxy",
"form.feed.label.disabled": "Do not refresh this feed",
"form.category.label.title": "Title",
"form.user.label.username": "Username",

1
locale/translations/es_ES.json
View File

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "Reglas de raspador",
"form.feed.label.rewrite_rules": "Reglas de reescribir",
"form.feed.label.ignore_http_cache": "Ignorar caché HTTP",
"form.feed.label.fetch_via_proxy": "Buscar a través de proxy",
"form.feed.label.disabled": "No actualice este feed",
"form.category.label.title": "Título",
"form.user.label.username": "Nombre de usuario",

1
locale/translations/fr_FR.json
View File

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "Règles pour récupérer le contenu original",
"form.feed.label.rewrite_rules": "Règles de réécriture",
"form.feed.label.ignore_http_cache": "Ignore cache HTTP",
"form.feed.label.fetch_via_proxy": "Récupérer via proxy",
"form.feed.label.disabled": "Ne pas actualiser ce flux",
"form.category.label.title": "Titre",
"form.user.label.username": "Nom d'utilisateur",

1
locale/translations/it_IT.json
View File

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "Regole di estrazione del contenuto",
"form.feed.label.rewrite_rules": "Regole di impaginazione del contenuto",
"form.feed.label.ignore_http_cache": "Ignora cache HTTP",
"form.feed.label.fetch_via_proxy": "Recuperare tramite proxy",
"form.feed.label.disabled": "Non aggiornare questo feed",
"form.category.label.title": "Titolo",
"form.user.label.username": "Nome utente",

1
locale/translations/ja_JP.json
View File

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "スクラップルール",
"form.feed.label.rewrite_rules": "Rewrite ルール",
"form.feed.label.ignore_http_cache": "HTTPキャッシュを無視",
"form.feed.label.fetch_via_proxy": "プロキシ経由でフェッチ",
"form.feed.label.disabled": "このフィードを更新しない",
"form.category.label.title": "タイトル",
"form.user.label.username": "ユーザー名",

1
locale/translations/nl_NL.json
View File

@ -249,6 +249,7 @@
"form.feed.label.scraper_rules": "Scraper regels",
"form.feed.label.rewrite_rules": "Rewrite regels",
"form.feed.label.ignore_http_cache": "Negeer HTTP-cache",
"form.feed.label.fetch_via_proxy": "Ophalen via proxy",
"form.feed.label.disabled": "Vernieuw deze feed niet",
"form.category.label.title": "Naam",
"form.user.label.username": "Gebruikersnaam",

1
locale/translations/pl_PL.json
View File

@ -251,6 +251,7 @@
"form.feed.label.scraper_rules": "Zasady ekstrakcji",
"form.feed.label.rewrite_rules": "Reguły zapisu",
"form.feed.label.ignore_http_cache": "Zignoruj ​​pamięć podręczną HTTP",
"form.feed.label.fetch_via_proxy": "Pobierz przez proxy",
"form.feed.label.disabled": "Не обновлять этот канал",
"form.category.label.title": "Tytuł",
"form.user.label.username": "Nazwa użytkownika",

1
locale/translations/pt_BR.json
View File

@ -250,6 +250,7 @@
"form.feed.label.rewrite_rules": "Regras para o Rewrite",
"form.feed.label.ignore_http_cache": "Ignorar cache HTTP",
"form.feed.label.disabled": "Não atualizar esta fonte",
"form.feed.label.fetch_via_proxy": "Buscar via proxy",
"form.category.label.title": "Título",
"form.user.label.username": "Nome de usuário",
"form.user.label.password": "Senha",

1
locale/translations/ru_RU.json
View File

@ -251,6 +251,7 @@
"form.feed.label.scraper_rules": "Правила Scraper",
"form.feed.label.rewrite_rules": "Правила Rewrite",
"form.feed.label.ignore_http_cache": "Игнорировать HTTP-кеш",
"form.feed.label.fetch_via_proxy": "Получить через прокси",
"form.feed.label.disabled": "Не обновлять этот канал",
"form.category.label.title": "Название",
"form.user.label.username": "Имя пользователя",

1
locale/translations/zh_CN.json
View File

@ -247,6 +247,7 @@
"form.feed.label.scraper_rules": "Scraper 规则",
"form.feed.label.rewrite_rules": "重写规则",
"form.feed.label.ignore_http_cache": "忽略HTTP缓存",
"form.feed.label.fetch_via_proxy": "通过代理获取",
"form.feed.label.disabled": "请勿刷新此Feed",
"form.category.label.title": "标题",
"form.user.label.username": "用户名",

5
miniflux.1
View File

@ -250,6 +250,11 @@ Maximum body size for HTTP requests in Mebibyte (MiB)\&.
.br
Default is 15 MiB\&.
.TP
.B HTTP_CLIENT_PROXY
Proxy URL for HTTP client\&.
.br
Default is empty\&.
.TP
.B AUTH_PROXY_HEADER
Proxy authentication HTTP header\&.
.TP

4
model/feed.go
View File

@ -34,6 +34,7 @@ type Feed struct {
Password string `json:"password"`
Disabled bool `json:"disabled"`
IgnoreHTTPCache bool `json:"ignore_http_cache"`
FetchViaProxy bool `json:"fetch_via_proxy"`
Category *Category `json:"category,omitempty"`
Entries Entries `json:"entries,omitempty"`
Icon *FeedIcon `json:"icon"`
@ -71,13 +72,14 @@ func (f *Feed) WithCategoryID(categoryID int64) {
}
// WithBrowsingParameters defines browsing parameters.
func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, password, scraperRules, rewriteRules string) {
func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, password, scraperRules, rewriteRules string, fetchViaProxy bool) {
f.Crawler = crawler
f.UserAgent = userAgent
f.Username = username
f.Password = password
f.ScraperRules = scraperRules
f.RewriteRules = rewriteRules
f.FetchViaProxy = fetchViaProxy
}
// WithError adds a new error message and increment the error counter.

2
model/feed_test.go
View File

@ -48,7 +48,7 @@ func TestFeedCategorySetter(t *testing.T) {
func TestFeedBrowsingParams(t *testing.T) {
feed := &Feed{}
feed.WithBrowsingParameters(true, "Custom User Agent", "Username", "Secret", "Some Rule", "Another Rule")
feed.WithBrowsingParameters(true, "Custom User Agent", "Username", "Secret", "Some Rule", "Another Rule", false)
if !feed.Crawler {
t.Error(`The crawler must be activated`)

21
reader/feed/handler.go
View File

@ -34,7 +34,7 @@ type Handler struct {
}
// CreateFeed fetch, parse and store a new feed.
func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password, scraperRules, rewriteRules string) (*model.Feed, error) {
func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password, scraperRules, rewriteRules string, fetchViaProxy bool) (*model.Feed, error) {
defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Handler:CreateFeed] feedUrl=%s", url))
if !h.store.CategoryExists(userID, categoryID) {
@ -44,6 +44,11 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool,
request := client.New(url)
request.WithCredentials(username, password)
request.WithUserAgent(userAgent)
if fetchViaProxy {
request.WithProxy()
}
response, requestErr := browser.Exec(request)
if requestErr != nil {
return nil, requestErr
@ -60,7 +65,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool,
subscription.UserID = userID
subscription.WithCategoryID(categoryID)
subscription.WithBrowsingParameters(crawler, userAgent, username, password, scraperRules, rewriteRules)
subscription.WithBrowsingParameters(crawler, userAgent, username, password, scraperRules, rewriteRules, fetchViaProxy)
subscription.WithClientResponse(response)
subscription.CheckedNow()
@ -72,7 +77,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool,
logger.Debug("[Handler:CreateFeed] Feed saved with ID: %d", subscription.ID)
checkFeedIcon(h.store, subscription.ID, subscription.SiteURL)
checkFeedIcon(h.store, subscription.ID, subscription.SiteURL, fetchViaProxy)
return subscription, nil
}
@ -111,6 +116,10 @@ func (h *Handler) RefreshFeed(userID, feedID int64) error {
request.WithCacheHeaders(originalFeed.EtagHeader, originalFeed.LastModifiedHeader)
}
if originalFeed.FetchViaProxy {
request.WithProxy()
}
response, requestErr := browser.Exec(request)
if requestErr != nil {
originalFeed.WithError(requestErr.Localize(printer))
@ -141,7 +150,7 @@ func (h *Handler) RefreshFeed(userID, feedID int64) error {
// We update caching headers only if the feed has been modified,
// because some websites don't return the same headers when replying with a 304.
originalFeed.WithClientResponse(response)
checkFeedIcon(h.store, originalFeed.ID, originalFeed.SiteURL)
checkFeedIcon(h.store, originalFeed.ID, originalFeed.SiteURL, originalFeed.FetchViaProxy)
} else {
logger.Debug("[Handler:RefreshFeed] Feed #%d not modified", feedID)
}
@ -162,9 +171,9 @@ func NewFeedHandler(store *storage.Storage) *Handler {
return &Handler{store}
}
func checkFeedIcon(store *storage.Storage, feedID int64, websiteURL string) {
func checkFeedIcon(store *storage.Storage, feedID int64, websiteURL string, fetchViaProxy bool) {
if !store.HasIcon(feedID) {
icon, err := icon.FindIcon(websiteURL)
icon, err := icon.FindIcon(websiteURL, fetchViaProxy)
if err != nil {
logger.Debug("CheckFeedIcon: %v (feedID=%d websiteURL=%s)", err, feedID, websiteURL)
} else if icon == nil {

12
reader/icon/finder.go
View File

@ -21,9 +21,12 @@ import (
)
// FindIcon try to find the website's icon.
func FindIcon(websiteURL string) (*model.Icon, error) {
func FindIcon(websiteURL string, fetchViaProxy bool) (*model.Icon, error) {
rootURL := url.RootURL(websiteURL)
clt := client.New(rootURL)
if fetchViaProxy {
clt.WithProxy()
}
response, err := clt.Get()
if err != nil {
return nil, fmt.Errorf("unable to download website index page: %v", err)
@ -43,7 +46,7 @@ func FindIcon(websiteURL string) (*model.Icon, error) {
}
logger.Debug("[FindIcon] Fetching icon => %s", iconURL)
icon, err := downloadIcon(iconURL)
icon, err := downloadIcon(iconURL, fetchViaProxy)
if err != nil {
return nil, err
}
@ -86,8 +89,11 @@ func parseDocument(websiteURL string, data io.Reader) (string, error) {
return iconURL, nil
}
func downloadIcon(iconURL string) (*model.Icon, error) {
func downloadIcon(iconURL string, fetchViaProxy bool) (*model.Icon, error) {
clt := client.New(iconURL)
if fetchViaProxy {
clt.WithProxy()
}
response, err := clt.Get()
if err != nil {
return nil, fmt.Errorf("unable to download iconURL: %v", err)

7
reader/subscription/finder.go
View File

@ -26,13 +26,18 @@ var (
)
// FindSubscriptions downloads and try to find one or more subscriptions from an URL.
func FindSubscriptions(websiteURL, userAgent, username, password string) (Subscriptions, *errors.LocalizedError) {
func FindSubscriptions(websiteURL, userAgent, username, password string, fetchViaProxy bool) (Subscriptions, *errors.LocalizedError) {
websiteURL = findYoutubeChannelFeed(websiteURL)
websiteURL = parseYoutubeVideoPage(websiteURL)
request := client.New(websiteURL)
request.WithCredentials(username, password)
request.WithUserAgent(userAgent)
if fetchViaProxy {
request.WithProxy()
}
response, err := browser.Exec(request)
if err != nil {
return nil, err

17
storage/feed.go
View File

@ -32,6 +32,7 @@ var feedListQuery = `
f.username,
f.password,
f.ignore_http_cache,
f.fetch_via_proxy,
f.disabled,
f.category_id,
c.title as category_title,
@ -133,6 +134,7 @@ func (s *Storage) FeedsByCategoryWithCounters(userID, categoryID int64) (model.F
f.username,
f.password,
f.ignore_http_cache,
f.fetch_via_proxy,
f.disabled,
f.category_id,
c.title as category_title,
@ -242,6 +244,7 @@ func (s *Storage) fetchFeeds(feedQuery, counterQuery string, args ...interface{}
&feed.Username,
&feed.Password,
&feed.IgnoreHTTPCache,
&feed.FetchViaProxy,
&feed.Disabled,
&feed.Category.ID,
&feed.Category.Title,
@ -326,6 +329,7 @@ func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) {
f.username,
f.password,
f.ignore_http_cache,
f.fetch_via_proxy,
f.disabled,
f.category_id,
c.title as category_title,
@ -357,6 +361,7 @@ func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) {
&feed.Username,
&feed.Password,
&feed.IgnoreHTTPCache,
&feed.FetchViaProxy,
&feed.Disabled,
&feed.Category.ID,
&feed.Category.Title,
@ -396,10 +401,11 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
password,
disabled,
scraper_rules,
rewrite_rules
rewrite_rules,
fetch_via_proxy
)
VALUES
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)
RETURNING
id
`
@ -419,6 +425,7 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
feed.Disabled,
feed.ScraperRules,
feed.RewriteRules,
feed.FetchViaProxy,
).Scan(&feed.ID)
if err != nil {
return fmt.Errorf(`store: unable to create feed %q: %v`, feed.FeedURL, err)
@ -462,9 +469,10 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
password=$15,
disabled=$16,
next_check_at=$17,
ignore_http_cache=$18
ignore_http_cache=$18,
fetch_via_proxy=$19
WHERE
id=$19 AND user_id=$20
id=$20 AND user_id=$21
`
_, err = s.db.Exec(query,
feed.FeedURL,
@ -485,6 +493,7 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
feed.Disabled,
feed.NextCheckAt,
feed.IgnoreHTTPCache,
feed.FetchViaProxy,
feed.ID,
feed.UserID,
)

2
template/common.go
View File

@ -519,7 +519,7 @@ SOFTWARE.
var templateCommonMapChecksums = map[string]string{
"entry_pagination": "cdca9cf12586e41e5355190b06d9168f57f77b85924d1e63b13524bc15abcbf6",
"feed_list": "30acc9ecc413811e73a1dad120b5d44e29564de3ba794fb07ee886b30addfb19",
"feed_list": "931e43d328a116318c510de5658c688cd940b934c86b6ec82a472e1f81e020ae",
"feed_menu": "318d8662dda5ca9dfc75b909c8461e79c86fb5082df1428f67aaf856f19f4b50",
"icons": "3dbe754a98f524a227111191d76b8c6944711b13613cc548ee9e9808fe0bffb4",
"item_meta": "8306adf3ef9966de3e3dc74ca1042e51d778b027ab8cf0a60a2e94a0115982dc",

3
template/html/add_subscription.html
View File

@ -30,6 +30,9 @@
<summary>{{ t "page.add_feed.legend.advanced_options" }}</summary>
<div class="details-content">
<label><input type="checkbox" name="crawler" value="1" {{ if .form.Crawler }}checked{{ end }}> {{ t "form.feed.label.crawler" }}</label>
{{ if .hasProxyConfigured }}
<label><input type="checkbox" name="fetch_via_proxy" value="1" {{ if .form.FetchViaProxy }}checked{{ end }}> {{ t "form.feed.label.fetch_via_proxy" }}</label>
{{ end }}
<label for="form-user-agent">{{ t "form.feed.label.user_agent" }}</label>
<input type="text" name="user_agent" id="form-user-agent" placeholder="{{ .defaultUserAgent }}" value="{{ .form.UserAgent }}" autocomplete="off">

3
template/html/choose_subscription.html
View File

@ -14,6 +14,9 @@
<input type="hidden" name="feed_password" value="{{ .form.Password }}">
<input type="hidden" name="scraper_rules" value="{{ .form.ScraperRules }}">
<input type="hidden" name="rewrite_rules" value="{{ .form.RewriteRules }}">
{{ if .form.FetchViaProxy }}
<input type="hidden" name="fetch_via_proxy" value="1">
{{ end }}
{{ if .form.Crawler }}
<input type="hidden" name="crawler" value="1">
{{ end }}

3
template/html/edit_feed.html
View File

@ -73,6 +73,9 @@
<label><input type="checkbox" name="crawler" value="1" {{ if .form.Crawler }}checked{{ end }}> {{ t "form.feed.label.crawler" }}</label>
<label><input type="checkbox" name="ignore_http_cache" value="1" {{ if .form.IgnoreHTTPCache }}checked{{ end }}> {{ t "form.feed.label.ignore_http_cache" }}</label>
{{ if .hasProxyConfigured }}
<label><input type="checkbox" name="fetch_via_proxy" value="1" {{ if .form.FetchViaProxy }}checked{{ end }}> {{ t "form.feed.label.fetch_via_proxy" }}</label>
{{ end }}
<label><input type="checkbox" name="disabled" value="1" {{ if .form.Disabled }}checked{{ end }}> {{ t "form.feed.label.disabled" }}</label>
<div class="buttons">

15
template/views.go
View File

@ -61,6 +61,9 @@ var templateViewsMap = map[string]string{
<summary>{{ t "page.add_feed.legend.advanced_options" }}</summary>
<div class="details-content">
<label><input type="checkbox" name="crawler" value="1" {{ if .form.Crawler }}checked{{ end }}> {{ t "form.feed.label.crawler" }}</label>
{{ if .hasProxyConfigured }}
<label><input type="checkbox" name="fetch_via_proxy" value="1" {{ if .form.FetchViaProxy }}checked{{ end }}> {{ t "form.feed.label.fetch_via_proxy" }}</label>
{{ end }}
<label for="form-user-agent">{{ t "form.feed.label.user_agent" }}</label>
<input type="text" name="user_agent" id="form-user-agent" placeholder="{{ .defaultUserAgent }}" value="{{ .form.UserAgent }}" autocomplete="off">
@ -380,6 +383,9 @@ var templateViewsMap = map[string]string{
<input type="hidden" name="feed_password" value="{{ .form.Password }}">
<input type="hidden" name="scraper_rules" value="{{ .form.ScraperRules }}">
<input type="hidden" name="rewrite_rules" value="{{ .form.RewriteRules }}">
{{ if .form.FetchViaProxy }}
<input type="hidden" name="fetch_via_proxy" value="1">
{{ end }}
{{ if .form.Crawler }}
<input type="hidden" name="crawler" value="1">
{{ end }}
@ -592,6 +598,9 @@ var templateViewsMap = map[string]string{
<label><input type="checkbox" name="crawler" value="1" {{ if .form.Crawler }}checked{{ end }}> {{ t "form.feed.label.crawler" }}</label>
<label><input type="checkbox" name="ignore_http_cache" value="1" {{ if .form.IgnoreHTTPCache }}checked{{ end }}> {{ t "form.feed.label.ignore_http_cache" }}</label>
{{ if .hasProxyConfigured }}
<label><input type="checkbox" name="fetch_via_proxy" value="1" {{ if .form.FetchViaProxy }}checked{{ end }}> {{ t "form.feed.label.fetch_via_proxy" }}</label>
{{ end }}
<label><input type="checkbox" name="disabled" value="1" {{ if .form.Disabled }}checked{{ end }}> {{ t "form.feed.label.disabled" }}</label>
<div class="buttons">
@ -1548,18 +1557,18 @@ var templateViewsMap = map[string]string{
var templateViewsMapChecksums = map[string]string{
"about": "4035658497363d7af7f79be83190404eb21ec633fe8ec636bdfc219d9fc78cfc",
"add_subscription": "0dbea93b6fc07423fa066122ad960c69616b829533371a2dbadec1e22d4f1ae0",
"add_subscription": "63961a83964acca354bc30eaae1f5e80f410ae4091af8da317380d4298f79032",
"api_keys": "27d401b31a72881d5232486ba17eb47edaf5246eaedce81de88698c15ebb2284",
"bookmark_entries": "892fe6cbf5a3301416dfb76e62935b495ca194275cfe113105a85b40ce7c200f",
"categories": "9dfc3cb7bb91c7750753fe962ee4540dd1843e5f75f9e0a575ee964f6f9923e9",
"category_entries": "8fa0e0b8f85e2572c40dee855b6d636207c3561086b234c93100673774c06746",
"category_feeds": "07154127087f9b127f7290abad6020c35ad9ceb2490b869120b7628bc4413808",
"choose_subscription": "84c9730cadd78e6ee5a6b4c499aab33acddb4324ac01924d33387543eec4d702",
"choose_subscription": "22109d760ea8079c491561d0106f773c885efbf66f87d81fcf8700218260d2a0",
"create_api_key": "5f74d4e92a6684927f5305096378c8be278159a5cd88ce652c7be3280a7d1685",
"create_category": "6b22b5ce51abf4e225e23a79f81be09a7fb90acb265e93a8faf9446dff74018d",
"create_user": "9b73a55233615e461d1f07d99ad1d4d3b54532588ab960097ba3e090c85aaf3a",
"edit_category": "b1c0b38f1b714c5d884edcd61e5b5295a5f1c8b71c469b35391e4dcc97cc6d36",
"edit_feed": "ff90b1883e2934e0236d530d8b778affe7665a6b08cdf9ae612c7e56469818ef",
"edit_feed": "7e86275f8e9325ddbffe79f6db871e58ad86d08c396e9b2ff8af69a09c4bf63b",
"edit_user": "c692db9de1a084c57b93e95a14b041d39bf489846cbb91fc982a62b72b77062a",
"entry": "c503dcf77de37090b9f05352bb9d99729085eec6e7bc22be94f2b4b244b4e48c",
"feed_entries": "ea5b88e3ad6b166d83b70e021d7b420d025f80decb6e24c79d13f8ce7c910b04",

3
ui/feed_edit.go
View File

@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui"
import (
"net/http"
"miniflux.app/config"
"miniflux.app/http/client"
"miniflux.app/http/request"
"miniflux.app/http/response/html"
@ -52,6 +53,7 @@ func (h *handler) showEditFeedPage(w http.ResponseWriter, r *http.Request) {
Username: feed.Username,
Password: feed.Password,
IgnoreHTTPCache: feed.IgnoreHTTPCache,
FetchViaProxy: feed.FetchViaProxy,
Disabled: feed.Disabled,
}
@ -65,6 +67,7 @@ func (h *handler) showEditFeedPage(w http.ResponseWriter, r *http.Request) {
view.Set("countUnread", h.store.CountUnreadEntries(user.ID))
view.Set("countErrorFeeds", h.store.CountErrorFeeds(user.ID))
view.Set("defaultUserAgent", client.DefaultUserAgent)
view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured())
html.OK(w, r, view.Render("edit_feed"))
}

3
ui/form/feed.go
View File

@ -25,6 +25,7 @@ type FeedForm struct {
Username string
Password string
IgnoreHTTPCache bool
FetchViaProxy bool
Disabled bool
}
@ -51,6 +52,7 @@ func (f FeedForm) Merge(feed *model.Feed) *model.Feed {
feed.Username = f.Username
feed.Password = f.Password
feed.IgnoreHTTPCache = f.IgnoreHTTPCache
feed.FetchViaProxy = f.FetchViaProxy
feed.Disabled = f.Disabled
return feed
}
@ -74,6 +76,7 @@ func NewFeedForm(r *http.Request) *FeedForm {
Username: r.FormValue("feed_username"),
Password: r.FormValue("feed_password"),
IgnoreHTTPCache: r.FormValue("ignore_http_cache") == "1",
FetchViaProxy: r.FormValue("fetch_via_proxy") == "1",
Disabled: r.FormValue("disabled") == "1",
}
}

34
ui/form/subscription.go
View File

@ -13,14 +13,15 @@ import (
// SubscriptionForm represents the subscription form.
type SubscriptionForm struct {
URL string
CategoryID int64
Crawler bool
UserAgent string
Username string
Password string
ScraperRules string
RewriteRules string
URL string
CategoryID int64
Crawler bool
FetchViaProxy bool
UserAgent string
Username string
Password string
ScraperRules string
RewriteRules string
}
// Validate makes sure the form values are valid.
@ -40,13 +41,14 @@ func NewSubscriptionForm(r *http.Request) *SubscriptionForm {
}
return &SubscriptionForm{
URL: r.FormValue("url"),
Crawler: r.FormValue("crawler") == "1",
CategoryID: int64(categoryID),
UserAgent: r.FormValue("user_agent"),
Username: r.FormValue("feed_username"),
Password: r.FormValue("feed_password"),
ScraperRules: r.FormValue("scraper_rules"),
RewriteRules: r.FormValue("rewrite_rules"),
URL: r.FormValue("url"),
Crawler: r.FormValue("crawler") == "1",
FetchViaProxy: r.FormValue("fetch_via_proxy") == "1",
CategoryID: int64(categoryID),
UserAgent: r.FormValue("user_agent"),
Username: r.FormValue("feed_username"),
Password: r.FormValue("feed_password"),
ScraperRules: r.FormValue("scraper_rules"),
RewriteRules: r.FormValue("rewrite_rules"),
}
}

6
ui/subscription_add.go
View File

@ -2,14 +2,15 @@
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package ui // import "miniflux.app/ui"
package ui // import "miniflux.app/ui"
import (
"net/http"
"miniflux.app/config"
"miniflux.app/http/client"
"miniflux.app/http/response/html"
"miniflux.app/http/request"
"miniflux.app/http/response/html"
"miniflux.app/ui/form"
"miniflux.app/ui/session"
"miniflux.app/ui/view"
@ -38,6 +39,7 @@ func (h *handler) showAddSubscriptionPage(w http.ResponseWriter, r *http.Request
view.Set("countErrorFeeds", h.store.CountErrorFeeds(user.ID))
view.Set("defaultUserAgent", client.DefaultUserAgent)
view.Set("form", &form.SubscriptionForm{CategoryID: 0})
view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured())
html.OK(w, r, view.Render("add_subscription"))
}

4
ui/subscription_bookmarklet.go
View File

@ -2,11 +2,12 @@
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package ui // import "miniflux.app/ui"
package ui // import "miniflux.app/ui"
import (
"net/http"
"miniflux.app/config"
"miniflux.app/http/client"
"miniflux.app/http/request"
"miniflux.app/http/response/html"
@ -40,6 +41,7 @@ func (h *handler) bookmarklet(w http.ResponseWriter, r *http.Request) {
view.Set("countUnread", h.store.CountUnreadEntries(user.ID))
view.Set("countErrorFeeds", h.store.CountErrorFeeds(user.ID))
view.Set("defaultUserAgent", client.DefaultUserAgent)
view.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured())
html.OK(w, r, view.Render("add_subscription"))
}

1
ui/subscription_choose.go
View File

@ -57,6 +57,7 @@ func (h *handler) showChooseSubscriptionPage(w http.ResponseWriter, r *http.Requ
subscriptionForm.Password,
subscriptionForm.ScraperRules,
subscriptionForm.RewriteRules,
subscriptionForm.FetchViaProxy,
)
if err != nil {
view.Set("form", subscriptionForm)

5
ui/subscription_submit.go
View File

@ -7,6 +7,7 @@ package ui // import "miniflux.app/ui"
import (
"net/http"
"miniflux.app/config"
"miniflux.app/http/client"
"miniflux.app/http/request"
"miniflux.app/http/response/html"
@ -40,6 +41,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) {
v.Set("countUnread", h.store.CountUnreadEntries(user.ID))
v.Set("countErrorFeeds", h.store.CountErrorFeeds(user.ID))
v.Set("defaultUserAgent", client.DefaultUserAgent)
v.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured())
subscriptionForm := form.NewSubscriptionForm(r)
if err := subscriptionForm.Validate(); err != nil {
@ -54,6 +56,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) {
subscriptionForm.UserAgent,
subscriptionForm.Username,
subscriptionForm.Password,
subscriptionForm.FetchViaProxy,
)
if findErr != nil {
logger.Error("[UI:SubmitSubscription] %s", findErr)
@ -82,6 +85,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) {
subscriptionForm.Password,
subscriptionForm.ScraperRules,
subscriptionForm.RewriteRules,
subscriptionForm.FetchViaProxy,
)
if err != nil {
v.Set("form", subscriptionForm)
@ -99,6 +103,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) {
v.Set("user", user)
v.Set("countUnread", h.store.CountUnreadEntries(user.ID))
v.Set("countErrorFeeds", h.store.CountErrorFeeds(user.ID))
v.Set("hasProxyConfigured", config.Opts.HasHTTPClientProxyConfigured())
html.OK(w, r, v.Render("choose_subscription"))
}