Rename alternative scheduler to entry_frequency

This commit is contained in:
Frédéric Guillot 2020-05-25 14:59:15 -07:00
parent cead85b165
commit 7e5157f218
8 changed files with 181 additions and 189 deletions

View File

@ -736,17 +736,17 @@ func TestDefautSchedulerCountBasedMaxIntervalValue(t *testing.T) {
t.Fatalf(`Parsing failure: %v`, err) t.Fatalf(`Parsing failure: %v`, err)
} }
expected := defaultSchedulerCountBasedMaxInterval expected := defaultSchedulerEntryFrequencyMaxInterval
result := opts.SchedulerCountBasedMaxInterval() result := opts.SchedulerEntryFrequencyMaxInterval()
if result != expected { if result != expected {
t.Fatalf(`Unexpected SCHEDULER_ENTRY_COUNT_BASED_MAX_INTERVAL value, got %v instead of %v`, result, expected) t.Fatalf(`Unexpected SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL value, got %v instead of %v`, result, expected)
} }
} }
func TestDefautSchedulerCountBasedMaxInterval(t *testing.T) { func TestDefautSchedulerCountBasedMaxInterval(t *testing.T) {
os.Clearenv() os.Clearenv()
os.Setenv("SCHEDULER_ENTRY_COUNT_BASED_MAX_INTERVAL", "30") os.Setenv("SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL", "30")
parser := NewParser() parser := NewParser()
opts, err := parser.ParseEnvironmentVariables() opts, err := parser.ParseEnvironmentVariables()
@ -755,10 +755,10 @@ func TestDefautSchedulerCountBasedMaxInterval(t *testing.T) {
} }
expected := 30 expected := 30
result := opts.SchedulerCountBasedMaxInterval() result := opts.SchedulerEntryFrequencyMaxInterval()
if result != expected { if result != expected {
t.Fatalf(`Unexpected SCHEDULER_ENTRY_COUNT_BASED_MAX_INTERVAL value, got %v instead of %v`, result, expected) t.Fatalf(`Unexpected SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL value, got %v instead of %v`, result, expected)
} }
} }
@ -771,17 +771,17 @@ func TestDefautSchedulerCountBasedMinIntervalValue(t *testing.T) {
t.Fatalf(`Parsing failure: %v`, err) t.Fatalf(`Parsing failure: %v`, err)
} }
expected := defaultSchedulerCountBasedMinInterval expected := defaultSchedulerEntryFrequencyMinInterval
result := opts.SchedulerCountBasedMinInterval() result := opts.SchedulerEntryFrequencyMinInterval()
if result != expected { if result != expected {
t.Fatalf(`Unexpected SCHEDULER_ENTRY_COUNT_BASED_MIN_INTERVAL value, got %v instead of %v`, result, expected) t.Fatalf(`Unexpected SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL value, got %v instead of %v`, result, expected)
} }
} }
func TestDefautSchedulerCountBasedMinInterval(t *testing.T) { func TestDefautSchedulerCountBasedMinInterval(t *testing.T) {
os.Clearenv() os.Clearenv()
os.Setenv("SCHEDULER_ENTRY_COUNT_BASED_MIN_INTERVAL", "30") os.Setenv("SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL", "30")
parser := NewParser() parser := NewParser()
opts, err := parser.ParseEnvironmentVariables() opts, err := parser.ParseEnvironmentVariables()
@ -790,10 +790,10 @@ func TestDefautSchedulerCountBasedMinInterval(t *testing.T) {
} }
expected := 30 expected := 30
result := opts.SchedulerCountBasedMinInterval() result := opts.SchedulerEntryFrequencyMinInterval()
if result != expected { if result != expected {
t.Fatalf(`Unexpected SCHEDULER_ENTRY_COUNT_BASED_MIN_INTERVAL value, got %v instead of %v`, result, expected) t.Fatalf(`Unexpected SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL value, got %v instead of %v`, result, expected)
} }
} }

View File

@ -23,8 +23,8 @@ const (
defaultPollingFrequency = 60 defaultPollingFrequency = 60
defaultBatchSize = 10 defaultBatchSize = 10
defaultPollingScheduler = "round_robin" defaultPollingScheduler = "round_robin"
defaultSchedulerCountBasedMinInterval = 5 defaultSchedulerEntryFrequencyMinInterval = 5
defaultSchedulerCountBasedMaxInterval = 24 * 60 defaultSchedulerEntryFrequencyMaxInterval = 24 * 60
defaultRunMigrations = false defaultRunMigrations = false
defaultDatabaseURL = "user=postgres password=postgres dbname=miniflux2 sslmode=disable" defaultDatabaseURL = "user=postgres password=postgres dbname=miniflux2 sslmode=disable"
defaultDatabaseMaxConns = 20 defaultDatabaseMaxConns = 20
@ -78,8 +78,8 @@ type Options struct {
pollingFrequency int pollingFrequency int
batchSize int batchSize int
pollingScheduler string pollingScheduler string
schedulerCountBasedMinInterval int schedulerEntryFrequencyMinInterval int
schedulerCountBasedMaxInterval int schedulerEntryFrequencyMaxInterval int
workerPoolSize int workerPoolSize int
createAdmin bool createAdmin bool
proxyImages string proxyImages string
@ -123,8 +123,8 @@ func NewOptions() *Options {
pollingFrequency: defaultPollingFrequency, pollingFrequency: defaultPollingFrequency,
batchSize: defaultBatchSize, batchSize: defaultBatchSize,
pollingScheduler: defaultPollingScheduler, pollingScheduler: defaultPollingScheduler,
schedulerCountBasedMinInterval: defaultSchedulerCountBasedMinInterval, schedulerEntryFrequencyMinInterval: defaultSchedulerEntryFrequencyMinInterval,
schedulerCountBasedMaxInterval: defaultSchedulerCountBasedMaxInterval, schedulerEntryFrequencyMaxInterval: defaultSchedulerEntryFrequencyMaxInterval,
workerPoolSize: defaultWorkerPoolSize, workerPoolSize: defaultWorkerPoolSize,
createAdmin: defaultCreateAdmin, createAdmin: defaultCreateAdmin,
proxyImages: defaultProxyImages, proxyImages: defaultProxyImages,
@ -242,19 +242,19 @@ func (o *Options) BatchSize() int {
return o.batchSize return o.batchSize
} }
// PollingScheduler returns the scheduler used for polling feeds // PollingScheduler returns the scheduler used for polling feeds.
func (o *Options) PollingScheduler() string { func (o *Options) PollingScheduler() string {
return o.pollingScheduler return o.pollingScheduler
} }
// SchedulerCountBasedMaxInterval returns the maximum interval in minutes for the count-based scheduler // SchedulerEntryFrequencyMaxInterval returns the maximum interval in minutes for the entry frequency scheduler.
func (o *Options) SchedulerCountBasedMaxInterval() int { func (o *Options) SchedulerEntryFrequencyMaxInterval() int {
return o.schedulerCountBasedMaxInterval return o.schedulerEntryFrequencyMaxInterval
} }
// SchedulerCountBasedMinInterval returns the minimum interval in minutes for the count-based scheduler // SchedulerEntryFrequencyMinInterval returns the minimum interval in minutes for the entry frequency scheduler.
func (o *Options) SchedulerCountBasedMinInterval() int { func (o *Options) SchedulerEntryFrequencyMinInterval() int {
return o.schedulerCountBasedMinInterval return o.schedulerEntryFrequencyMinInterval
} }
// IsOAuth2UserCreationAllowed returns true if user creation is allowed for OAuth2 users. // IsOAuth2UserCreationAllowed returns true if user creation is allowed for OAuth2 users.
@ -374,8 +374,8 @@ func (o *Options) String() string {
builder.WriteString(fmt.Sprintf("POLLING_FREQUENCY: %v\n", o.pollingFrequency)) builder.WriteString(fmt.Sprintf("POLLING_FREQUENCY: %v\n", o.pollingFrequency))
builder.WriteString(fmt.Sprintf("BATCH_SIZE: %v\n", o.batchSize)) builder.WriteString(fmt.Sprintf("BATCH_SIZE: %v\n", o.batchSize))
builder.WriteString(fmt.Sprintf("POLLING_SCHEDULER: %v\n", o.pollingScheduler)) builder.WriteString(fmt.Sprintf("POLLING_SCHEDULER: %v\n", o.pollingScheduler))
builder.WriteString(fmt.Sprintf("SCHEDULER_ENTRY_COUNT_BASED_MAX_INTERVAL: %v\n", o.schedulerCountBasedMaxInterval)) builder.WriteString(fmt.Sprintf("SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL: %v\n", o.schedulerEntryFrequencyMaxInterval))
builder.WriteString(fmt.Sprintf("SCHEDULER_ENTRY_COUNT_BASED_MIN_INTERVAL: %v\n", o.schedulerCountBasedMinInterval)) builder.WriteString(fmt.Sprintf("SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL: %v\n", o.schedulerEntryFrequencyMinInterval))
builder.WriteString(fmt.Sprintf("PROXY_IMAGES: %v\n", o.proxyImages)) builder.WriteString(fmt.Sprintf("PROXY_IMAGES: %v\n", o.proxyImages))
builder.WriteString(fmt.Sprintf("CREATE_ADMIN: %v\n", o.createAdmin)) builder.WriteString(fmt.Sprintf("CREATE_ADMIN: %v\n", o.createAdmin))
builder.WriteString(fmt.Sprintf("POCKET_CONSUMER_KEY: %v\n", o.pocketConsumerKey)) builder.WriteString(fmt.Sprintf("POCKET_CONSUMER_KEY: %v\n", o.pocketConsumerKey))

View File

@ -139,11 +139,11 @@ func (p *Parser) parseLines(lines []string) (err error) {
case "BATCH_SIZE": case "BATCH_SIZE":
p.opts.batchSize = parseInt(value, defaultBatchSize) p.opts.batchSize = parseInt(value, defaultBatchSize)
case "POLLING_SCHEDULER": case "POLLING_SCHEDULER":
p.opts.pollingScheduler = parseString(value, defaultPollingScheduler) p.opts.pollingScheduler = strings.ToLower(parseString(value, defaultPollingScheduler))
case "SCHEDULER_ENTRY_COUNT_BASED_MAX_INTERVAL": case "SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL":
p.opts.schedulerCountBasedMaxInterval = parseInt(value, defaultSchedulerCountBasedMaxInterval) p.opts.schedulerEntryFrequencyMaxInterval = parseInt(value, defaultSchedulerEntryFrequencyMaxInterval)
case "SCHEDULER_ENTRY_COUNT_BASED_MIN_INTERVAL": case "SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL":
p.opts.schedulerCountBasedMinInterval = parseInt(value, defaultSchedulerCountBasedMinInterval) p.opts.schedulerEntryFrequencyMinInterval = parseInt(value, defaultSchedulerEntryFrequencyMinInterval)
case "PROXY_IMAGES": case "PROXY_IMAGES":
p.opts.proxyImages = parseString(value, defaultProxyImages) p.opts.proxyImages = parseString(value, defaultProxyImages)
case "CREATE_ADMIN": case "CREATE_ADMIN":

View File

@ -111,13 +111,13 @@ Refresh interval in minutes for feeds (default is 60 minutes)\&.
Number of feeds to send to the queue for each interval (default is 10)\&. Number of feeds to send to the queue for each interval (default is 10)\&.
.TP .TP
.B POLLING_SCHEDULER .B POLLING_SCHEDULER
The scheduler used for polling feeds. Possible values include: "round_robin", "entry_count_based" Scheduler used for polling feeds. Possible values are "round_robin" (default) or "entry_frequency"\&.
.TP .TP
.B SCHEDULER_ENTRY_COUNT_BASED_MAX_INTERVAL .B SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL
The maximum interval in minutes for the entry-count-based scheduler Maximum interval in minutes for the entry frequency scheduler (default is 24 hours)\&.
.TP .TP
.B SCHEDULER_ENTRY_COUNT_BASED_MIN_INTERVAL .B SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL
The minimum interval in minutes for the entry-count-based scheduler Minimum interval in minutes for the entry frequency scheduler (default is 5 minutes)\&.
.TP .TP
.B DATABASE_URL .B DATABASE_URL
Postgresql connection parameters\&. Postgresql connection parameters\&.

View File

@ -7,7 +7,6 @@ package model // import "miniflux.app/model"
import ( import (
"fmt" "fmt"
"math" "math"
"strings"
"time" "time"
"miniflux.app/config" "miniflux.app/config"
@ -41,9 +40,10 @@ type Feed struct {
ReadCount int `json:"-"` ReadCount int `json:"-"`
} }
// List of supported schedulers.
const ( const (
// SchedulerEntryCountBased represnets the name of the scheduler based on entry counts. SchedulerRoundRobin = "round_robin"
SchedulerEntryCountBased = "entry_count_based" SchedulerEntryFrequency = "entry_frequency"
) )
func (f *Feed) String() string { func (f *Feed) String() string {
@ -102,24 +102,20 @@ func (f *Feed) CheckedNow() {
// ScheduleNextCheck set "next_check_at" of a feed based on the scheduler selected from the configuration. // ScheduleNextCheck set "next_check_at" of a feed based on the scheduler selected from the configuration.
func (f *Feed) ScheduleNextCheck(weeklyCount int) { func (f *Feed) ScheduleNextCheck(weeklyCount int) {
var nextCheckAt time.Time switch config.Opts.PollingScheduler() {
switch strings.ToLower(config.Opts.PollingScheduler()) { case SchedulerEntryFrequency:
case SchedulerEntryCountBased:
var intervalMinutes int var intervalMinutes int
if weeklyCount == 0 { if weeklyCount == 0 {
intervalMinutes = config.Opts.SchedulerCountBasedMaxInterval() intervalMinutes = config.Opts.SchedulerEntryFrequencyMaxInterval()
} else { } else {
intervalMinutes = int(math.Round(float64(7*24*60) / float64(weeklyCount))) intervalMinutes = int(math.Round(float64(7*24*60) / float64(weeklyCount)))
} }
intervalMinutes = int(math.Min(float64(intervalMinutes), float64(config.Opts.SchedulerCountBasedMaxInterval()))) intervalMinutes = int(math.Min(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMaxInterval())))
intervalMinutes = int(math.Max(float64(intervalMinutes), float64(config.Opts.SchedulerCountBasedMinInterval()))) intervalMinutes = int(math.Max(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMinInterval())))
nextCheckAt = time.Now().Add(time.Minute * time.Duration(intervalMinutes)) f.NextCheckAt = time.Now().Add(time.Minute * time.Duration(intervalMinutes))
default: default:
// round robin f.NextCheckAt = time.Now()
// omit the interval because they are same for all feeds.
nextCheckAt = time.Now()
} }
f.NextCheckAt = nextCheckAt
} }
// Feeds is a list of feed // Feeds is a list of feed

View File

@ -133,9 +133,9 @@ func TestFeedScheduleNextCheckEntryCountBasedMaxInterval(t *testing.T) {
maxInterval := 5 maxInterval := 5
minInterval := 1 minInterval := 1
os.Clearenv() os.Clearenv()
os.Setenv("POLLING_SCHEDULER", "entry_count_based") os.Setenv("POLLING_SCHEDULER", "entry_frequency")
os.Setenv("SCHEDULER_ENTRY_COUNT_BASED_MAX_INTERVAL", fmt.Sprintf("%d", maxInterval)) os.Setenv("SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL", fmt.Sprintf("%d", maxInterval))
os.Setenv("SCHEDULER_ENTRY_COUNT_BASED_MIN_INTERVAL", fmt.Sprintf("%d", minInterval)) os.Setenv("SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL", fmt.Sprintf("%d", minInterval))
var err error var err error
parser := config.NewParser() parser := config.NewParser()
@ -160,9 +160,9 @@ func TestFeedScheduleNextCheckEntryCountBasedMinInterval(t *testing.T) {
maxInterval := 500 maxInterval := 500
minInterval := 100 minInterval := 100
os.Clearenv() os.Clearenv()
os.Setenv("POLLING_SCHEDULER", "entry_count_based") os.Setenv("POLLING_SCHEDULER", "entry_frequency")
os.Setenv("SCHEDULER_ENTRY_COUNT_BASED_MAX_INTERVAL", fmt.Sprintf("%d", maxInterval)) os.Setenv("SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL", fmt.Sprintf("%d", maxInterval))
os.Setenv("SCHEDULER_ENTRY_COUNT_BASED_MIN_INTERVAL", fmt.Sprintf("%d", minInterval)) os.Setenv("SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL", fmt.Sprintf("%d", minInterval))
var err error var err error
parser := config.NewParser() parser := config.NewParser()

View File

@ -8,6 +8,7 @@ import (
"fmt" "fmt"
"time" "time"
"miniflux.app/config"
"miniflux.app/errors" "miniflux.app/errors"
"miniflux.app/http/client" "miniflux.app/http/client"
"miniflux.app/locale" "miniflux.app/locale"
@ -90,13 +91,17 @@ func (h *Handler) RefreshFeed(userID, feedID int64) error {
return errors.NewLocalizedError(errNotFound, feedID) return errors.NewLocalizedError(errNotFound, feedID)
} }
weeklyCount, parametersErr := h.store.FeedSchedulerParameters(userID, feedID) weeklyEntryCount := 0
if parametersErr != nil { if config.Opts.PollingScheduler() == model.SchedulerEntryFrequency {
return parametersErr var weeklyCountErr error
weeklyEntryCount, weeklyCountErr = h.store.WeeklyFeedEntryCount(userID, feedID)
if weeklyCountErr != nil {
return weeklyCountErr
}
} }
originalFeed.CheckedNow() originalFeed.CheckedNow()
originalFeed.ScheduleNextCheck(weeklyCount) originalFeed.ScheduleNextCheck(weeklyEntryCount)
request := client.New(originalFeed.FeedURL) request := client.New(originalFeed.FeedURL)
request.WithCredentials(originalFeed.Username, originalFeed.Password) request.WithCredentials(originalFeed.Username, originalFeed.Password)

View File

@ -8,9 +8,7 @@ import (
"database/sql" "database/sql"
"errors" "errors"
"fmt" "fmt"
"strings"
"miniflux.app/config"
"miniflux.app/model" "miniflux.app/model"
"miniflux.app/timezone" "miniflux.app/timezone"
) )
@ -274,14 +272,8 @@ func (s *Storage) fetchFeeds(feedQuery, counterQuery string, args ...interface{}
return feeds, nil return feeds, nil
} }
// FeedSchedulerParameters returns the parameters used for the scheduler. // WeeklyFeedEntryCount returns the weekly entry count for a feed.
func (s *Storage) FeedSchedulerParameters(userID, feedID int64) (int, error) { func (s *Storage) WeeklyFeedEntryCount(userID, feedID int64) (int, error) {
scheduler := strings.ToLower(config.Opts.PollingScheduler())
if scheduler != model.SchedulerEntryCountBased {
return 0, nil
}
var weeklyCount int
query := ` query := `
SELECT SELECT
count(*) count(*)
@ -293,15 +285,14 @@ func (s *Storage) FeedSchedulerParameters(userID, feedID int64) (int, error) {
entries.published_at BETWEEN (now() - interval '1 week') AND now(); entries.published_at BETWEEN (now() - interval '1 week') AND now();
` `
err := s.db.QueryRow(query, userID, feedID).Scan( var weeklyCount int
&weeklyCount, err := s.db.QueryRow(query, userID, feedID).Scan(&weeklyCount)
)
switch { switch {
case err == sql.ErrNoRows: case err == sql.ErrNoRows:
return 0, nil return 0, nil
case err != nil: case err != nil:
return 0, fmt.Errorf(`store: unable to fetch scheduler parameters for feed #%d: %v`, feedID, err) return 0, fmt.Errorf(`store: unable to fetch weekly count for feed #%d: %v`, feedID, err)
} }
return weeklyCount, nil return weeklyCount, nil