diff --git a/internal/api/category.go b/internal/api/category.go index 9e13a28f..7b47e2a3 100644 --- a/internal/api/category.go +++ b/internal/api/category.go @@ -9,6 +9,7 @@ import ( "net/http" "time" + "miniflux.app/v2/internal/config" "miniflux.app/v2/internal/http/request" "miniflux.app/v2/internal/http/response/json" "miniflux.app/v2/internal/model" @@ -136,7 +137,14 @@ func (h *handler) refreshCategory(w http.ResponseWriter, r *http.Request) { userID := request.UserID(r) categoryID := request.RouteInt64Param(r, "categoryID") - jobs, err := h.store.NewCategoryBatch(userID, categoryID, h.store.CountFeeds(userID)) + batchBuilder := h.store.NewBatchBuilder() + batchBuilder.WithErrorLimit(config.Opts.PollingParsingErrorLimit()) + batchBuilder.WithoutDisabledFeeds() + batchBuilder.WithUserID(userID) + batchBuilder.WithCategoryID(categoryID) + batchBuilder.WithNextCheckExpired() + + jobs, err := batchBuilder.FetchJobs() if err != nil { json.ServerError(w, r, err) return diff --git a/internal/api/feed.go b/internal/api/feed.go index 0f486f70..27f93ab7 100644 --- a/internal/api/feed.go +++ b/internal/api/feed.go @@ -9,6 +9,7 @@ import ( "net/http" "time" + "miniflux.app/v2/internal/config" "miniflux.app/v2/internal/http/request" "miniflux.app/v2/internal/http/response/json" "miniflux.app/v2/internal/model" @@ -69,7 +70,14 @@ func (h *handler) refreshFeed(w http.ResponseWriter, r *http.Request) { func (h *handler) refreshAllFeeds(w http.ResponseWriter, r *http.Request) { userID := request.UserID(r) - jobs, err := h.store.NewUserBatch(userID, h.store.CountFeeds(userID)) + + batchBuilder := h.store.NewBatchBuilder() + batchBuilder.WithErrorLimit(config.Opts.PollingParsingErrorLimit()) + batchBuilder.WithoutDisabledFeeds() + batchBuilder.WithNextCheckExpired() + batchBuilder.WithUserID(userID) + + jobs, err := batchBuilder.FetchJobs() if err != nil { json.ServerError(w, r, err) return diff --git a/internal/cli/refresh_feeds.go b/internal/cli/refresh_feeds.go index 3af4ecdb..e10600f3 100644 --- a/internal/cli/refresh_feeds.go +++ b/internal/cli/refresh_feeds.go @@ -18,7 +18,15 @@ func refreshFeeds(store *storage.Storage) { var wg sync.WaitGroup startTime := time.Now() - jobs, err := store.NewBatch(config.Opts.BatchSize()) + + // Generate a batch of feeds for any user that has feeds to refresh. + batchBuilder := store.NewBatchBuilder() + batchBuilder.WithBatchSize(config.Opts.BatchSize()) + batchBuilder.WithErrorLimit(config.Opts.PollingParsingErrorLimit()) + batchBuilder.WithoutDisabledFeeds() + batchBuilder.WithNextCheckExpired() + + jobs, err := batchBuilder.FetchJobs() if err != nil { slog.Error("Unable to fetch jobs from database", slog.Any("error", err)) return diff --git a/internal/cli/scheduler.go b/internal/cli/scheduler.go index 6bde37c0..9f69d7ea 100644 --- a/internal/cli/scheduler.go +++ b/internal/cli/scheduler.go @@ -20,6 +20,7 @@ func runScheduler(store *storage.Storage, pool *worker.Pool) { pool, config.Opts.PollingFrequency(), config.Opts.BatchSize(), + config.Opts.PollingParsingErrorLimit(), ) go cleanupScheduler( @@ -28,10 +29,16 @@ func runScheduler(store *storage.Storage, pool *worker.Pool) { ) } -func feedScheduler(store *storage.Storage, pool *worker.Pool, frequency, batchSize int) { +func feedScheduler(store *storage.Storage, pool *worker.Pool, frequency, batchSize, errorLimit int) { for range time.Tick(time.Duration(frequency) * time.Minute) { - jobs, err := store.NewBatch(batchSize) - if err != nil { + // Generate a batch of feeds for any user that has feeds to refresh. + batchBuilder := store.NewBatchBuilder() + batchBuilder.WithBatchSize(batchSize) + batchBuilder.WithErrorLimit(errorLimit) + batchBuilder.WithoutDisabledFeeds() + batchBuilder.WithNextCheckExpired() + + if jobs, err := batchBuilder.FetchJobs(); err != nil { slog.Error("Unable to fetch jobs from database", slog.Any("error", err)) } else { slog.Info("Created a batch of feeds", diff --git a/internal/http/request/context.go b/internal/http/request/context.go index 7fb50685..9a0acbf4 100644 --- a/internal/http/request/context.go +++ b/internal/http/request/context.go @@ -3,7 +3,10 @@ package request // import "miniflux.app/v2/internal/http/request" -import "net/http" +import ( + "net/http" + "strconv" +) // ContextKey represents a context key. type ContextKey int @@ -24,6 +27,7 @@ const ( FlashMessageContextKey FlashErrorMessageContextKey PocketRequestTokenContextKey + LastForceRefreshContextKey ClientIPContextKey GoogleReaderToken ) @@ -114,6 +118,16 @@ func PocketRequestToken(r *http.Request) string { return getContextStringValue(r, PocketRequestTokenContextKey) } +// LastForceRefresh returns the last force refresh timestamp. +func LastForceRefresh(r *http.Request) int64 { + jsonStringValue := getContextStringValue(r, LastForceRefreshContextKey) + timestamp, err := strconv.ParseInt(jsonStringValue, 10, 64) + if err != nil { + return 0 + } + return timestamp +} + // ClientIP returns the client IP address stored in the context. func ClientIP(r *http.Request) string { return getContextStringValue(r, ClientIPContextKey) diff --git a/internal/locale/translations/de_DE.json b/internal/locale/translations/de_DE.json index 25a1b776..15851aa4 100644 --- a/internal/locale/translations/de_DE.json +++ b/internal/locale/translations/de_DE.json @@ -453,6 +453,6 @@ "You are not authorized to access this resource (invalid username/password)": "Sie sind nicht berechtigt, auf diese Ressource zuzugreifen (Benutzername/Passwort ungültig)", "Unable to fetch this resource (Status Code = %d)": "Ressource konnte nicht abgerufen werden (code=%d)", "Resource not found (404), this feed doesn't exist anymore, check the feed URL": "Ressource nicht gefunden (404), dieses Abonnement existiert nicht mehr, überprüfen Sie die Abonnement-URL", - "page.background_feed_refresh.title": "Hintergrundaktualisierung", + "alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.", "alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running." } diff --git a/internal/locale/translations/el_EL.json b/internal/locale/translations/el_EL.json index 5915c510..cbacf37b 100644 --- a/internal/locale/translations/el_EL.json +++ b/internal/locale/translations/el_EL.json @@ -434,6 +434,6 @@ "πριν %d έτος", "πριν %d έτη" ], - "page.background_feed_refresh.title": "Background feed refresh", + "alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.", "alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running." } diff --git a/internal/locale/translations/en_US.json b/internal/locale/translations/en_US.json index 73983c70..89f16821 100644 --- a/internal/locale/translations/en_US.json +++ b/internal/locale/translations/en_US.json @@ -434,6 +434,6 @@ "%d year ago", "%d years ago" ], - "page.background_feed_refresh.title": "Background feed refresh", + "alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.", "alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running." } diff --git a/internal/locale/translations/es_ES.json b/internal/locale/translations/es_ES.json index 1718377b..3e9198b8 100644 --- a/internal/locale/translations/es_ES.json +++ b/internal/locale/translations/es_ES.json @@ -434,6 +434,6 @@ "hace %d año", "hace %d años" ], - "page.background_feed_refresh.title": "Background feed refresh", + "alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.", "alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running." } diff --git a/internal/locale/translations/fi_FI.json b/internal/locale/translations/fi_FI.json index 5e8287f8..2a950314 100644 --- a/internal/locale/translations/fi_FI.json +++ b/internal/locale/translations/fi_FI.json @@ -434,6 +434,6 @@ "%d vuosi sitten", "%d vuotta sitten" ], - "page.background_feed_refresh.title": "Background feed refresh", + "alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.", "alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running." } diff --git a/internal/locale/translations/fr_FR.json b/internal/locale/translations/fr_FR.json index d2b90ca4..0b3b27d2 100644 --- a/internal/locale/translations/fr_FR.json +++ b/internal/locale/translations/fr_FR.json @@ -453,6 +453,6 @@ "You are not authorized to access this resource (invalid username/password)": "Vous n'êtes pas autorisé à accéder à cette ressource (nom d'utilisateur / mot de passe incorrect)", "Unable to fetch this resource (Status Code = %d)": "Impossible de récupérer cette ressource (code=%d)", "Resource not found (404), this feed doesn't exist anymore, check the feed URL": "Page introuvable (404), cet abonnement n'existe plus, vérifiez l'adresse du flux", - "page.background_feed_refresh.title": "Actualisation des abonnements en arrière-plan", + "alert.too_many_feeds_refresh": "Vous avez déclenché trop d'actualisations de flux. Veuillez attendre 30 minutes avant de réessayer.", "alert.background_feed_refresh": "Les abonnements sont en cours d'actualisation en arrière-plan. Vous pouvez continuer à naviguer dans l'application." } diff --git a/internal/locale/translations/hi_IN.json b/internal/locale/translations/hi_IN.json index 2aa402d1..50cf7753 100644 --- a/internal/locale/translations/hi_IN.json +++ b/internal/locale/translations/hi_IN.json @@ -434,6 +434,6 @@ "%d साल पहले", "%d वर्षों पहले" ], - "page.background_feed_refresh.title": "फ़ीड रीफ़्रेश किया जा रहा है", + "alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.", "alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running." } diff --git a/internal/locale/translations/id_ID.json b/internal/locale/translations/id_ID.json index 320e3e92..8d3c7f9a 100644 --- a/internal/locale/translations/id_ID.json +++ b/internal/locale/translations/id_ID.json @@ -444,6 +444,6 @@ "You are not authorized to access this resource (invalid username/password)": "Anda tidak memiliki izin yang cukup untuk mengakses umpan ini (nama pengguna/kata sandi tidak valid)", "Unable to fetch this resource (Status Code = %d)": "Tidak bisa mengambil umpan ini (Kode Status = %d)", "Resource not found (404), this feed doesn't exist anymore, check the feed URL": "Umpan tidak ditemukan (404), umpan ini tidak ada lagi, periksa URL umpan", - "page.background_feed_refresh.title": "Memuat ulang umpan di latar belakang", + "alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.", "alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running." } diff --git a/internal/locale/translations/it_IT.json b/internal/locale/translations/it_IT.json index d82edc1c..13ad90f2 100644 --- a/internal/locale/translations/it_IT.json +++ b/internal/locale/translations/it_IT.json @@ -434,6 +434,6 @@ "%d anno fa", "%d anni fa" ], - "page.background_feed_refresh.title": "Aggiornamento in background", + "alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.", "alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running." } diff --git a/internal/locale/translations/ja_JP.json b/internal/locale/translations/ja_JP.json index 58b4a018..3bae562d 100644 --- a/internal/locale/translations/ja_JP.json +++ b/internal/locale/translations/ja_JP.json @@ -434,6 +434,6 @@ "%d 年前", "%d 年前" ], - "page.background_feed_refresh.title": "バックグラウンドでフィードを更新中", + "alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.", "alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running." } diff --git a/internal/locale/translations/nl_NL.json b/internal/locale/translations/nl_NL.json index 79df2226..ddc280d6 100644 --- a/internal/locale/translations/nl_NL.json +++ b/internal/locale/translations/nl_NL.json @@ -451,6 +451,6 @@ "Invalid SSL certificate (original error: %q)": "Ongeldig SSL-certificaat (originele error: %q)", "This website is unreachable (original error: %q)": "Deze website is onbereikbaar (originele error: %q)", "Website unreachable, the request timed out after %d seconds": "Website onbereikbaar, de request gaf een timeout na %d seconden", - "page.background_feed_refresh.title": "Achtergrond vernieuwen", + "alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.", "alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running." } diff --git a/internal/locale/translations/pl_PL.json b/internal/locale/translations/pl_PL.json index 9d1bd812..e27614d6 100644 --- a/internal/locale/translations/pl_PL.json +++ b/internal/locale/translations/pl_PL.json @@ -459,6 +459,6 @@ "Invalid SSL certificate (original error: %q)": "Certyfikat SSL jest nieprawidłowy (błąd: %q)", "This website is unreachable (original error: %q)": "Ta strona jest niedostępna (błąd: %q)", "Website unreachable, the request timed out after %d seconds": "Strona internetowa nieosiągalna, żądanie wygasło po %d sekundach", - "page.background_feed_refresh.title": "Odświeżanie kanałów w tle", + "alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.", "alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running." } diff --git a/internal/locale/translations/pt_BR.json b/internal/locale/translations/pt_BR.json index 821f5fae..ce019f4e 100644 --- a/internal/locale/translations/pt_BR.json +++ b/internal/locale/translations/pt_BR.json @@ -434,6 +434,6 @@ "há %d ano", "há %d anos" ], - "page.background_feed_refresh.title": "Atualização de fonte em segundo plano", + "alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.", "alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running." } diff --git a/internal/locale/translations/ru_RU.json b/internal/locale/translations/ru_RU.json index c328d2ff..fd542255 100644 --- a/internal/locale/translations/ru_RU.json +++ b/internal/locale/translations/ru_RU.json @@ -442,6 +442,6 @@ "%d года назад", "%d лет назад" ], - "page.background_feed_refresh.title": "Обновление подписок", + "alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.", "alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running." } diff --git a/internal/locale/translations/tr_TR.json b/internal/locale/translations/tr_TR.json index 840bfe55..4515c569 100644 --- a/internal/locale/translations/tr_TR.json +++ b/internal/locale/translations/tr_TR.json @@ -434,6 +434,6 @@ "%d yıl önce", "%d yıl önce" ], - "page.background_feed_refresh.title": "Arka plan beslemesi yenileme", + "alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.", "alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running." } diff --git a/internal/locale/translations/uk_UA.json b/internal/locale/translations/uk_UA.json index 49e9e708..c7a7b2b0 100644 --- a/internal/locale/translations/uk_UA.json +++ b/internal/locale/translations/uk_UA.json @@ -443,6 +443,6 @@ "%d роки тому", "%d років тому" ], - "page.background_feed_refresh.title": "Оновлення стрічок в фоновому режимі", + "alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.", "alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running." } diff --git a/internal/locale/translations/zh_CN.json b/internal/locale/translations/zh_CN.json index a77b55fc..63697de0 100644 --- a/internal/locale/translations/zh_CN.json +++ b/internal/locale/translations/zh_CN.json @@ -444,6 +444,6 @@ "Invalid SSL certificate (original error: %q)": "无效的 SSL 证书 (原始错误: %q)", "This website is unreachable (original error: %q)": "该网站永久不可达 (原始错误: %q)", "Website unreachable, the request timed out after %d seconds": "网站不可达, 请求已在 %d 秒后超时", - "page.background_feed_refresh.title": "后台更新", + "alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.", "alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running." } diff --git a/internal/locale/translations/zh_TW.json b/internal/locale/translations/zh_TW.json index 77bf7a36..d7da6d62 100644 --- a/internal/locale/translations/zh_TW.json +++ b/internal/locale/translations/zh_TW.json @@ -452,6 +452,6 @@ "Invalid SSL certificate (original error: %q)": "無效的 SSL 憑證 (錯誤: %q)", "This website is unreachable (original error: %q)": "該網站永久無法訪問(原始錯誤: %q)", "Website unreachable, the request timed out after %d seconds": "網站無法訪問, 請求已在 %d 秒後超時", - "page.background_feed_refresh.title": "背景更新", + "alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.", "alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running." } diff --git a/internal/model/app_session.go b/internal/model/app_session.go index 44c1e251..a2fed4c1 100644 --- a/internal/model/app_session.go +++ b/internal/model/app_session.go @@ -20,11 +20,21 @@ type SessionData struct { Language string `json:"language"` Theme string `json:"theme"` PocketRequestToken string `json:"pocket_request_token"` + LastForceRefresh string `json:"last_force_refresh"` } func (s SessionData) String() string { - return fmt.Sprintf(`CSRF=%q, OAuth2State=%q, OAuth2CodeVerifier=%q, FlashMsg=%q, FlashErrMsg=%q, Lang=%q, Theme=%q, PocketTkn=%q`, - s.CSRF, s.OAuth2State, s.OAuth2CodeVerifier, s.FlashMessage, s.FlashErrorMessage, s.Language, s.Theme, s.PocketRequestToken) + return fmt.Sprintf(`CSRF=%q, OAuth2State=%q, OAuth2CodeVerifier=%q, FlashMsg=%q, FlashErrMsg=%q, Lang=%q, Theme=%q, PocketTkn=%q, LastForceRefresh=%s`, + s.CSRF, + s.OAuth2State, + s.OAuth2CodeVerifier, + s.FlashMessage, + s.FlashErrorMessage, + s.Language, + s.Theme, + s.PocketRequestToken, + s.LastForceRefresh, + ) } // Value converts the session data to JSON. diff --git a/internal/storage/batch.go b/internal/storage/batch.go new file mode 100644 index 00000000..107d480e --- /dev/null +++ b/internal/storage/batch.go @@ -0,0 +1,91 @@ +// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package storage // import "miniflux.app/v2/internal/storage" + +import ( + "database/sql" + "fmt" + "strings" + + "miniflux.app/v2/internal/model" +) + +type BatchBuilder struct { + db *sql.DB + args []any + conditions []string + limit int +} + +func (s *Storage) NewBatchBuilder() *BatchBuilder { + return &BatchBuilder{ + db: s.db, + } +} + +func (b *BatchBuilder) WithBatchSize(batchSize int) *BatchBuilder { + b.limit = batchSize + return b +} + +func (b *BatchBuilder) WithUserID(userID int64) *BatchBuilder { + b.conditions = append(b.conditions, fmt.Sprintf("user_id = $%d", len(b.args)+1)) + b.args = append(b.args, userID) + return b +} + +func (b *BatchBuilder) WithCategoryID(categoryID int64) *BatchBuilder { + b.conditions = append(b.conditions, fmt.Sprintf("category_id = $%d", len(b.args)+1)) + b.args = append(b.args, categoryID) + return b +} + +func (b *BatchBuilder) WithErrorLimit(limit int) *BatchBuilder { + if limit > 0 { + b.conditions = append(b.conditions, fmt.Sprintf("parsing_error_count < $%d", len(b.args)+1)) + b.args = append(b.args, limit) + } + return b +} + +func (b *BatchBuilder) WithNextCheckExpired() *BatchBuilder { + b.conditions = append(b.conditions, "next_check_at < now()") + return b +} + +func (b *BatchBuilder) WithoutDisabledFeeds() *BatchBuilder { + b.conditions = append(b.conditions, "disabled is false") + return b +} + +func (b *BatchBuilder) FetchJobs() (jobs model.JobList, err error) { + var parts []string + parts = append(parts, `SELECT id, user_id FROM feeds`) + + if len(b.conditions) > 0 { + parts = append(parts, fmt.Sprintf("WHERE %s", strings.Join(b.conditions, " AND "))) + } + + if b.limit > 0 { + parts = append(parts, fmt.Sprintf("ORDER BY next_check_at ASC LIMIT %d", b.limit)) + } + + query := strings.Join(parts, " ") + rows, err := b.db.Query(query, b.args...) + if err != nil { + return nil, fmt.Errorf(`store: unable to fetch batch of jobs: %v`, err) + } + defer rows.Close() + + for rows.Next() { + var job model.Job + if err := rows.Scan(&job.FeedID, &job.UserID); err != nil { + return nil, fmt.Errorf(`store: unable to fetch job: %v`, err) + } + + jobs = append(jobs, job) + } + + return jobs, nil +} diff --git a/internal/storage/feed.go b/internal/storage/feed.go index b7f52e65..f365591e 100644 --- a/internal/storage/feed.go +++ b/internal/storage/feed.go @@ -87,17 +87,6 @@ func (s *Storage) CountAllFeeds() map[string]int64 { return results } -// CountFeeds returns the number of feeds that belongs to the given user. -func (s *Storage) CountFeeds(userID int64) int { - var result int - err := s.db.QueryRow(`SELECT count(*) FROM feeds WHERE user_id=$1`, userID).Scan(&result) - if err != nil { - return 0 - } - - return result -} - // CountUserFeedsWithErrors returns the number of feeds with parsing errors that belong to the given user. func (s *Storage) CountUserFeedsWithErrors(userID int64) int { pollingParsingErrorLimit := config.Opts.PollingParsingErrorLimit() diff --git a/internal/storage/job.go b/internal/storage/job.go deleted file mode 100644 index a4b355a0..00000000 --- a/internal/storage/job.go +++ /dev/null @@ -1,81 +0,0 @@ -// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package storage // import "miniflux.app/v2/internal/storage" - -import ( - "fmt" - - "miniflux.app/v2/internal/config" - "miniflux.app/v2/internal/model" -) - -// NewBatch returns a series of jobs. -func (s *Storage) NewBatch(batchSize int) (jobs model.JobList, err error) { - pollingParsingErrorLimit := config.Opts.PollingParsingErrorLimit() - query := ` - SELECT - id, - user_id - FROM - feeds - WHERE - disabled is false AND next_check_at < now() AND - CASE WHEN $1 > 0 THEN parsing_error_count < $1 ELSE parsing_error_count >= 0 END - ORDER BY next_check_at ASC LIMIT $2 - ` - return s.fetchBatchRows(query, pollingParsingErrorLimit, batchSize) -} - -// NewUserBatch returns a series of jobs but only for a given user. -func (s *Storage) NewUserBatch(userID int64, batchSize int) (jobs model.JobList, err error) { - // We do not take the error counter into consideration when the given - // user refresh manually all his feeds to force a refresh. - query := ` - SELECT - id, - user_id - FROM - feeds - WHERE - user_id=$1 AND disabled is false AND next_check_at < now() - ORDER BY next_check_at ASC LIMIT %d - ` - return s.fetchBatchRows(fmt.Sprintf(query, batchSize), userID) -} - -// NewCategoryBatch returns a series of jobs but only for a given category. -func (s *Storage) NewCategoryBatch(userID int64, categoryID int64, batchSize int) (jobs model.JobList, err error) { - // We do not take the error counter into consideration when the given - // user refresh manually all his feeds to force a refresh. - query := ` - SELECT - id, - user_id - FROM - feeds - WHERE - user_id=$1 AND category_id=$2 AND disabled is false AND next_check_at < now() - ORDER BY next_check_at ASC LIMIT %d - ` - return s.fetchBatchRows(fmt.Sprintf(query, batchSize), userID, categoryID) -} - -func (s *Storage) fetchBatchRows(query string, args ...interface{}) (jobs model.JobList, err error) { - rows, err := s.db.Query(query, args...) - if err != nil { - return nil, fmt.Errorf(`store: unable to fetch batch of jobs: %v`, err) - } - defer rows.Close() - - for rows.Next() { - var job model.Job - if err := rows.Scan(&job.FeedID, &job.UserID); err != nil { - return nil, fmt.Errorf(`store: unable to fetch job: %v`, err) - } - - jobs = append(jobs, job) - } - - return jobs, nil -} diff --git a/internal/template/templates/views/feed_background_refresh.html b/internal/template/templates/views/feed_background_refresh.html deleted file mode 100644 index b1b6a6d0..00000000 --- a/internal/template/templates/views/feed_background_refresh.html +++ /dev/null @@ -1,11 +0,0 @@ -{{ define "title"}}{{ t "page.background_feed_refresh.title" }}{{ end }} - -{{ define "content"}} -
-

{{ t "page.background_feed_refresh.title" }}

- {{ template "feed_menu" }} -
- -

{{ t "alert.background_feed_refresh" }}

- -{{ end }} \ No newline at end of file diff --git a/internal/ui/category_refresh.go b/internal/ui/category_refresh.go index f852a64b..adbf015a 100644 --- a/internal/ui/category_refresh.go +++ b/internal/ui/category_refresh.go @@ -6,10 +6,13 @@ package ui // import "miniflux.app/v2/internal/ui" import ( "log/slog" "net/http" + "time" "miniflux.app/v2/internal/http/request" "miniflux.app/v2/internal/http/response/html" "miniflux.app/v2/internal/http/route" + "miniflux.app/v2/internal/locale" + "miniflux.app/v2/internal/ui/session" ) func (h *handler) refreshCategoryEntriesPage(w http.ResponseWriter, r *http.Request) { @@ -25,21 +28,38 @@ func (h *handler) refreshCategoryFeedsPage(w http.ResponseWriter, r *http.Reques func (h *handler) refreshCategory(w http.ResponseWriter, r *http.Request) int64 { userID := request.UserID(r) categoryID := request.RouteInt64Param(r, "categoryID") + printer := locale.NewPrinter(request.UserLanguage(r)) + sess := session.New(h.store, request.SessionID(r)) - jobs, err := h.store.NewCategoryBatch(userID, categoryID, h.store.CountFeeds(userID)) - if err != nil { - html.ServerError(w, r, err) - return 0 + // Avoid accidental and excessive refreshes. + if time.Now().UTC().Unix()-request.LastForceRefresh(r) < 1800 { + sess.NewFlashErrorMessage(printer.Printf("alert.too_many_feeds_refresh")) + } else { + // We allow the end-user to force refresh all its feeds in this category + // without taking into consideration the number of errors. + batchBuilder := h.store.NewBatchBuilder() + batchBuilder.WithoutDisabledFeeds() + batchBuilder.WithUserID(userID) + batchBuilder.WithCategoryID(categoryID) + + jobs, err := batchBuilder.FetchJobs() + if err != nil { + html.ServerError(w, r, err) + return 0 + } + + slog.Info( + "Triggered a manual refresh of all feeds for a given category from the web ui", + slog.Int64("user_id", userID), + slog.Int64("category_id", categoryID), + slog.Int("nb_jobs", len(jobs)), + ) + + go h.pool.Push(jobs) + + sess.SetLastForceRefresh() + sess.NewFlashMessage(printer.Printf("alert.background_feed_refresh")) } - slog.Info( - "Triggered a manual refresh of all feeds for a given category from the web ui", - slog.Int64("user_id", userID), - slog.Int64("category_id", categoryID), - slog.Int("nb_jobs", len(jobs)), - ) - - go h.pool.Push(jobs) - return categoryID } diff --git a/internal/ui/feed_refresh.go b/internal/ui/feed_refresh.go index 57ad3ac4..c07e082a 100644 --- a/internal/ui/feed_refresh.go +++ b/internal/ui/feed_refresh.go @@ -6,13 +6,14 @@ package ui // import "miniflux.app/v2/internal/ui" import ( "log/slog" "net/http" + "time" "miniflux.app/v2/internal/http/request" "miniflux.app/v2/internal/http/response/html" "miniflux.app/v2/internal/http/route" + "miniflux.app/v2/internal/locale" feedHandler "miniflux.app/v2/internal/reader/handler" "miniflux.app/v2/internal/ui/session" - "miniflux.app/v2/internal/ui/view" ) func (h *handler) refreshFeed(w http.ResponseWriter, r *http.Request) { @@ -32,33 +33,36 @@ func (h *handler) refreshFeed(w http.ResponseWriter, r *http.Request) { func (h *handler) refreshAllFeeds(w http.ResponseWriter, r *http.Request) { userID := request.UserID(r) - - user, err := h.store.UserByID(userID) - if err != nil { - html.ServerError(w, r, err) - return - } - - jobs, err := h.store.NewUserBatch(userID, h.store.CountFeeds(userID)) - if err != nil { - html.ServerError(w, r, err) - return - } - - slog.Info( - "Triggered a manual refresh of all feeds from the web ui", - slog.Int64("user_id", userID), - slog.Int("nb_jobs", len(jobs)), - ) - - go h.pool.Push(jobs) - + printer := locale.NewPrinter(request.UserLanguage(r)) sess := session.New(h.store, request.SessionID(r)) - view := view.New(h.tpl, r, sess) - view.Set("menu", "feeds") - view.Set("user", user) - view.Set("countUnread", h.store.CountUnreadEntries(user.ID)) - view.Set("countErrorFeeds", h.store.CountUserFeedsWithErrors(user.ID)) - html.OK(w, r, view.Render("feed_background_refresh")) + // Avoid accidental and excessive refreshes. + if time.Now().UTC().Unix()-request.LastForceRefresh(r) < 1800 { + sess.NewFlashErrorMessage(printer.Printf("alert.too_many_feeds_refresh")) + } else { + // We allow the end-user to force refresh all its feeds + // without taking into consideration the number of errors. + batchBuilder := h.store.NewBatchBuilder() + batchBuilder.WithoutDisabledFeeds() + batchBuilder.WithUserID(userID) + + jobs, err := batchBuilder.FetchJobs() + if err != nil { + html.ServerError(w, r, err) + return + } + + slog.Info( + "Triggered a manual refresh of all feeds from the web ui", + slog.Int64("user_id", userID), + slog.Int("nb_jobs", len(jobs)), + ) + + go h.pool.Push(jobs) + + sess.SetLastForceRefresh() + sess.NewFlashMessage(printer.Printf("alert.background_feed_refresh")) + } + + html.Redirect(w, r, route.Path(h.router, "feeds")) } diff --git a/internal/ui/middleware.go b/internal/ui/middleware.go index 7df1c6af..7cfa5b34 100644 --- a/internal/ui/middleware.go +++ b/internal/ui/middleware.go @@ -119,6 +119,8 @@ func (m *middleware) handleAppSession(next http.Handler) http.Handler { ctx = context.WithValue(ctx, request.UserLanguageContextKey, session.Data.Language) ctx = context.WithValue(ctx, request.UserThemeContextKey, session.Data.Theme) ctx = context.WithValue(ctx, request.PocketRequestTokenContextKey, session.Data.PocketRequestToken) + ctx = context.WithValue(ctx, request.LastForceRefreshContextKey, session.Data.LastForceRefresh) + next.ServeHTTP(w, r.WithContext(ctx)) }) } diff --git a/internal/ui/session/session.go b/internal/ui/session/session.go index 619c383a..c47a1828 100644 --- a/internal/ui/session/session.go +++ b/internal/ui/session/session.go @@ -4,6 +4,8 @@ package session // import "miniflux.app/v2/internal/ui/session" import ( + "time" + "miniflux.app/v2/internal/storage" ) @@ -13,6 +15,15 @@ type Session struct { sessionID string } +// New returns a new session handler. +func New(store *storage.Storage, sessionID string) *Session { + return &Session{store, sessionID} +} + +func (s *Session) SetLastForceRefresh() { + s.store.UpdateAppSessionField(s.sessionID, "last_force_refresh", time.Now().UTC().Unix()) +} + func (s *Session) SetOAuth2State(state string) { s.store.UpdateAppSessionField(s.sessionID, "oauth2_state", state) } @@ -61,8 +72,3 @@ func (s *Session) SetTheme(theme string) { func (s *Session) SetPocketRequestToken(requestToken string) { s.store.UpdateAppSessionField(s.sessionID, "pocket_request_token", requestToken) } - -// New returns a new session handler. -func New(store *storage.Storage, sessionID string) *Session { - return &Session{store, sessionID} -}