From bea9017b4833ee8608657feb31d650fe48183c8d Mon Sep 17 00:00:00 2001 From: Dror Levin Date: Thu, 24 Aug 2023 12:02:46 +0200 Subject: [PATCH] Add factor for entry_frequency scheduler Allow the user to increase the frequency of the entry_frequency scheduler by a configurable factor in order to shorten the time between updates. --- internal/config/config_test.go | 35 ++++++++++++++++++++++++++++++++++ internal/config/options.go | 9 +++++++++ internal/config/parser.go | 2 ++ internal/model/feed.go | 6 +++--- internal/model/feed_test.go | 25 ++++++++++++++++++++++++ miniflux.1 | 5 +++++ 6 files changed, 79 insertions(+), 3 deletions(-) diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 596d640b..d10c4471 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -724,6 +724,41 @@ func TestDefautSchedulerCountBasedMinInterval(t *testing.T) { } } +func TestDefautSchedulerEntryFrequencyFactorValue(t *testing.T) { + os.Clearenv() + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := defaultSchedulerEntryFrequencyFactor + result := opts.SchedulerEntryFrequencyFactor() + + if result != expected { + t.Fatalf(`Unexpected SCHEDULER_ENTRY_FREQUENCY_FACTOR value, got %v instead of %v`, result, expected) + } +} + +func TestDefautSchedulerEntryFrequencyFactor(t *testing.T) { + os.Clearenv() + os.Setenv("SCHEDULER_ENTRY_FREQUENCY_FACTOR", "2") + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := 2 + result := opts.SchedulerEntryFrequencyFactor() + + if result != expected { + t.Fatalf(`Unexpected SCHEDULER_ENTRY_FREQUENCY_FACTOR value, got %v instead of %v`, result, expected) + } +} + func TestPollingParsingErrorLimit(t *testing.T) { os.Clearenv() os.Setenv("POLLING_PARSING_ERROR_LIMIT", "100") diff --git a/internal/config/options.go b/internal/config/options.go index 5ca6306c..d4a56d4b 100644 --- a/internal/config/options.go +++ b/internal/config/options.go @@ -30,6 +30,7 @@ const ( defaultPollingScheduler = "round_robin" defaultSchedulerEntryFrequencyMinInterval = 5 defaultSchedulerEntryFrequencyMaxInterval = 24 * 60 + defaultSchedulerEntryFrequencyFactor = 1 defaultPollingParsingErrorLimit = 3 defaultRunMigrations = false defaultDatabaseURL = "user=postgres password=postgres dbname=miniflux2 sslmode=disable" @@ -118,6 +119,7 @@ type Options struct { pollingScheduler string schedulerEntryFrequencyMinInterval int schedulerEntryFrequencyMaxInterval int + schedulerEntryFrequencyFactor int pollingParsingErrorLimit int workerPoolSize int createAdmin bool @@ -191,6 +193,7 @@ func NewOptions() *Options { pollingScheduler: defaultPollingScheduler, schedulerEntryFrequencyMinInterval: defaultSchedulerEntryFrequencyMinInterval, schedulerEntryFrequencyMaxInterval: defaultSchedulerEntryFrequencyMaxInterval, + schedulerEntryFrequencyFactor: defaultSchedulerEntryFrequencyFactor, pollingParsingErrorLimit: defaultPollingParsingErrorLimit, workerPoolSize: defaultWorkerPoolSize, createAdmin: defaultCreateAdmin, @@ -368,6 +371,11 @@ func (o *Options) SchedulerEntryFrequencyMinInterval() int { return o.schedulerEntryFrequencyMinInterval } +// SchedulerEntryFrequencyFactor returns the factor for the entry frequency scheduler. +func (o *Options) SchedulerEntryFrequencyFactor() int { + return o.schedulerEntryFrequencyFactor +} + // PollingParsingErrorLimit returns the limit of errors when to stop polling. func (o *Options) PollingParsingErrorLimit() int { return o.pollingParsingErrorLimit @@ -628,6 +636,7 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option { "RUN_MIGRATIONS": o.runMigrations, "SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL": o.schedulerEntryFrequencyMaxInterval, "SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL": o.schedulerEntryFrequencyMinInterval, + "SCHEDULER_ENTRY_FREQUENCY_FACTOR": o.schedulerEntryFrequencyFactor, "SCHEDULER_SERVICE": o.schedulerService, "SERVER_TIMING_HEADER": o.serverTimingHeader, "WATCHDOG": o.watchdog, diff --git a/internal/config/parser.go b/internal/config/parser.go index c754de5c..bb43bafa 100644 --- a/internal/config/parser.go +++ b/internal/config/parser.go @@ -135,6 +135,8 @@ func (p *Parser) parseLines(lines []string) (err error) { p.opts.schedulerEntryFrequencyMaxInterval = parseInt(value, defaultSchedulerEntryFrequencyMaxInterval) case "SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL": p.opts.schedulerEntryFrequencyMinInterval = parseInt(value, defaultSchedulerEntryFrequencyMinInterval) + case "SCHEDULER_ENTRY_FREQUENCY_FACTOR": + p.opts.schedulerEntryFrequencyFactor = parseInt(value, defaultSchedulerEntryFrequencyFactor) case "POLLING_PARSING_ERROR_LIMIT": p.opts.pollingParsingErrorLimit = parseInt(value, defaultPollingParsingErrorLimit) // kept for compatibility purpose diff --git a/internal/model/feed.go b/internal/model/feed.go index 4add53b6..64269567 100644 --- a/internal/model/feed.go +++ b/internal/model/feed.go @@ -115,10 +115,10 @@ func (f *Feed) ScheduleNextCheck(weeklyCount int) { if weeklyCount == 0 { intervalMinutes = config.Opts.SchedulerEntryFrequencyMaxInterval() } else { - intervalMinutes = int(math.Round(float64(7*24*60) / float64(weeklyCount))) + intervalMinutes = int(math.Round(float64(7*24*60) / float64(weeklyCount*config.Opts.SchedulerEntryFrequencyFactor()))) + intervalMinutes = int(math.Min(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMaxInterval()))) + intervalMinutes = int(math.Max(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMinInterval()))) } - intervalMinutes = int(math.Min(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMaxInterval()))) - intervalMinutes = int(math.Max(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMinInterval()))) f.NextCheckAt = time.Now().Add(time.Minute * time.Duration(intervalMinutes)) default: f.NextCheckAt = time.Now() diff --git a/internal/model/feed_test.go b/internal/model/feed_test.go index 91850f50..34f89e90 100644 --- a/internal/model/feed_test.go +++ b/internal/model/feed_test.go @@ -152,3 +152,28 @@ func TestFeedScheduleNextCheckEntryCountBasedMinInterval(t *testing.T) { t.Error(`The next_check_at should not be before the now + min interval`) } } + +func TestFeedScheduleNextCheckEntryFrequencyFactor(t *testing.T) { + factor := 2 + os.Clearenv() + os.Setenv("POLLING_SCHEDULER", "entry_frequency") + os.Setenv("SCHEDULER_ENTRY_FREQUENCY_FACTOR", fmt.Sprintf("%d", factor)) + + var err error + parser := config.NewParser() + config.Opts, err = parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + feed := &Feed{} + weeklyCount := 7 + feed.ScheduleNextCheck(weeklyCount) + + if feed.NextCheckAt.IsZero() { + t.Error(`The next_check_at must be set`) + } + + if feed.NextCheckAt.After(time.Now().Add(time.Minute * time.Duration(config.Opts.SchedulerEntryFrequencyMaxInterval()/factor))) { + t.Error(`The next_check_at should not be after the now + factor * count`) + } +} diff --git a/miniflux.1 b/miniflux.1 index 1d66a8c4..7ae9b507 100644 --- a/miniflux.1 +++ b/miniflux.1 @@ -181,6 +181,11 @@ Minimum interval in minutes for the entry frequency scheduler\&. .br Default is 5 minutes\&. .TP +.B SCHEDULER_ENTRY_FREQUENCY_FACTOR +Factor to increase refresh frequency for the entry frequency scheduler\&. +.br +Default is 1\&. +.TP .B POLLING_PARSING_ERROR_LIMIT The maximum number of parsing errors that the program will try before stopping polling a feed. Once the limit is reached, the user must refresh the feed manually. Set to 0 for unlimited. .br