forks/miniflux
1
0
Fork 0
Code Releases Activity

Rename alternative scheduler to entry_frequency

Browse source
This commit is contained in:
Frédéric Guillot 2020-05-25 14:59:15 -07:00
parent cead85b165
commit 7e5157f218
8 changed files with 181 additions and 189 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
config/config_test.go
View file

@ -736,17 +736,17 @@ func TestDefautSchedulerCountBasedMaxIntervalValue(t *testing.T) {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := defaultSchedulerCountBasedMaxInterval
result := opts.SchedulerCountBasedMaxInterval()
expected := defaultSchedulerEntryFrequencyMaxInterval
result := opts.SchedulerEntryFrequencyMaxInterval()
if result != expected {
t.Fatalf(`Unexpected SCHEDULER_ENTRY_COUNT_BASED_MAX_INTERVAL value, got %v instead of %v`, result, expected)
t.Fatalf(`Unexpected SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL value, got %v instead of %v`, result, expected)
}
}
func TestDefautSchedulerCountBasedMaxInterval(t *testing.T) {
os.Clearenv()
os.Setenv("SCHEDULER_ENTRY_COUNT_BASED_MAX_INTERVAL", "30")
os.Setenv("SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL", "30")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
@ -755,10 +755,10 @@ func TestDefautSchedulerCountBasedMaxInterval(t *testing.T) {
}
expected := 30
result := opts.SchedulerCountBasedMaxInterval()
result := opts.SchedulerEntryFrequencyMaxInterval()
if result != expected {
t.Fatalf(`Unexpected SCHEDULER_ENTRY_COUNT_BASED_MAX_INTERVAL value, got %v instead of %v`, result, expected)
t.Fatalf(`Unexpected SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL value, got %v instead of %v`, result, expected)
}
}
@ -771,17 +771,17 @@ func TestDefautSchedulerCountBasedMinIntervalValue(t *testing.T) {
t.Fatalf(`Parsing failure: %v`, err)
}
expected := defaultSchedulerCountBasedMinInterval
result := opts.SchedulerCountBasedMinInterval()
expected := defaultSchedulerEntryFrequencyMinInterval
result := opts.SchedulerEntryFrequencyMinInterval()
if result != expected {
t.Fatalf(`Unexpected SCHEDULER_ENTRY_COUNT_BASED_MIN_INTERVAL value, got %v instead of %v`, result, expected)
t.Fatalf(`Unexpected SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL value, got %v instead of %v`, result, expected)
}
}
func TestDefautSchedulerCountBasedMinInterval(t *testing.T) {
os.Clearenv()
os.Setenv("SCHEDULER_ENTRY_COUNT_BASED_MIN_INTERVAL", "30")
os.Setenv("SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL", "30")
parser := NewParser()
opts, err := parser.ParseEnvironmentVariables()
@ -790,10 +790,10 @@ func TestDefautSchedulerCountBasedMinInterval(t *testing.T) {
}
expected := 30
result := opts.SchedulerCountBasedMinInterval()
result := opts.SchedulerEntryFrequencyMinInterval()
if result != expected {
t.Fatalf(`Unexpected SCHEDULER_ENTRY_COUNT_BASED_MIN_INTERVAL value, got %v instead of %v`, result, expected)
t.Fatalf(`Unexpected SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL value, got %v instead of %v`, result, expected)
}
}

30
config/options.go
View file

@ -23,8 +23,8 @@ const (
defaultPollingFrequency = 60
defaultBatchSize = 10
defaultPollingScheduler = "round_robin"
defaultSchedulerCountBasedMinInterval = 5
defaultSchedulerCountBasedMaxInterval = 24 * 60
defaultSchedulerEntryFrequencyMinInterval = 5
defaultSchedulerEntryFrequencyMaxInterval = 24 * 60
defaultRunMigrations = false
defaultDatabaseURL = "user=postgres password=postgres dbname=miniflux2 sslmode=disable"
defaultDatabaseMaxConns = 20
@ -78,8 +78,8 @@ type Options struct {
pollingFrequency int
batchSize int
pollingScheduler string
schedulerCountBasedMinInterval int
schedulerCountBasedMaxInterval int
schedulerEntryFrequencyMinInterval int
schedulerEntryFrequencyMaxInterval int
workerPoolSize int
createAdmin bool
proxyImages string
@ -123,8 +123,8 @@ func NewOptions() *Options {
pollingFrequency: defaultPollingFrequency,
batchSize: defaultBatchSize,
pollingScheduler: defaultPollingScheduler,
schedulerCountBasedMinInterval: defaultSchedulerCountBasedMinInterval,
schedulerCountBasedMaxInterval: defaultSchedulerCountBasedMaxInterval,
schedulerEntryFrequencyMinInterval: defaultSchedulerEntryFrequencyMinInterval,
schedulerEntryFrequencyMaxInterval: defaultSchedulerEntryFrequencyMaxInterval,
workerPoolSize: defaultWorkerPoolSize,
createAdmin: defaultCreateAdmin,
proxyImages: defaultProxyImages,
@ -242,19 +242,19 @@ func (o *Options) BatchSize() int {
return o.batchSize
}
// PollingScheduler returns the scheduler used for polling feeds
// PollingScheduler returns the scheduler used for polling feeds.
func (o *Options) PollingScheduler() string {
return o.pollingScheduler
}
// SchedulerCountBasedMaxInterval returns the maximum interval in minutes for the count-based scheduler
func (o *Options) SchedulerCountBasedMaxInterval() int {
return o.schedulerCountBasedMaxInterval
// SchedulerEntryFrequencyMaxInterval returns the maximum interval in minutes for the entry frequency scheduler.
func (o *Options) SchedulerEntryFrequencyMaxInterval() int {
return o.schedulerEntryFrequencyMaxInterval
}
// SchedulerCountBasedMinInterval returns the minimum interval in minutes for the count-based scheduler
func (o *Options) SchedulerCountBasedMinInterval() int {
return o.schedulerCountBasedMinInterval
// SchedulerEntryFrequencyMinInterval returns the minimum interval in minutes for the entry frequency scheduler.
func (o *Options) SchedulerEntryFrequencyMinInterval() int {
return o.schedulerEntryFrequencyMinInterval
}
// IsOAuth2UserCreationAllowed returns true if user creation is allowed for OAuth2 users.
@ -374,8 +374,8 @@ func (o *Options) String() string {
builder.WriteString(fmt.Sprintf("POLLING_FREQUENCY: %v\n", o.pollingFrequency))
builder.WriteString(fmt.Sprintf("BATCH_SIZE: %v\n", o.batchSize))
builder.WriteString(fmt.Sprintf("POLLING_SCHEDULER: %v\n", o.pollingScheduler))
builder.WriteString(fmt.Sprintf("SCHEDULER_ENTRY_COUNT_BASED_MAX_INTERVAL: %v\n", o.schedulerCountBasedMaxInterval))
builder.WriteString(fmt.Sprintf("SCHEDULER_ENTRY_COUNT_BASED_MIN_INTERVAL: %v\n", o.schedulerCountBasedMinInterval))
builder.WriteString(fmt.Sprintf("SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL: %v\n", o.schedulerEntryFrequencyMaxInterval))
builder.WriteString(fmt.Sprintf("SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL: %v\n", o.schedulerEntryFrequencyMinInterval))
builder.WriteString(fmt.Sprintf("PROXY_IMAGES: %v\n", o.proxyImages))
builder.WriteString(fmt.Sprintf("CREATE_ADMIN: %v\n", o.createAdmin))
builder.WriteString(fmt.Sprintf("POCKET_CONSUMER_KEY: %v\n", o.pocketConsumerKey))

10
config/parser.go
View file

@ -139,11 +139,11 @@ func (p *Parser) parseLines(lines []string) (err error) {
case "BATCH_SIZE":
p.opts.batchSize = parseInt(value, defaultBatchSize)
case "POLLING_SCHEDULER":
p.opts.pollingScheduler = parseString(value, defaultPollingScheduler)
case "SCHEDULER_ENTRY_COUNT_BASED_MAX_INTERVAL":
p.opts.schedulerCountBasedMaxInterval = parseInt(value, defaultSchedulerCountBasedMaxInterval)
case "SCHEDULER_ENTRY_COUNT_BASED_MIN_INTERVAL":
p.opts.schedulerCountBasedMinInterval = parseInt(value, defaultSchedulerCountBasedMinInterval)
p.opts.pollingScheduler = strings.ToLower(parseString(value, defaultPollingScheduler))
case "SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL":
p.opts.schedulerEntryFrequencyMaxInterval = parseInt(value, defaultSchedulerEntryFrequencyMaxInterval)
case "SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL":
p.opts.schedulerEntryFrequencyMinInterval = parseInt(value, defaultSchedulerEntryFrequencyMinInterval)
case "PROXY_IMAGES":
p.opts.proxyImages = parseString(value, defaultProxyImages)
case "CREATE_ADMIN":

10
miniflux.1
View file

@ -111,13 +111,13 @@ Refresh interval in minutes for feeds (default is 60 minutes)\&.
Number of feeds to send to the queue for each interval (default is 10)\&.
.TP
.B POLLING_SCHEDULER
The scheduler used for polling feeds. Possible values include: "round_robin", "entry_count_based"
Scheduler used for polling feeds. Possible values are "round_robin" (default) or "entry_frequency"\&.
.TP
.B SCHEDULER_ENTRY_COUNT_BASED_MAX_INTERVAL
The maximum interval in minutes for the entry-count-based scheduler
.B SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL
Maximum interval in minutes for the entry frequency scheduler (default is 24 hours)\&.
.TP
.B SCHEDULER_ENTRY_COUNT_BASED_MIN_INTERVAL
The minimum interval in minutes for the entry-count-based scheduler
.B SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL
Minimum interval in minutes for the entry frequency scheduler (default is 5 minutes)\&.
.TP
.B DATABASE_URL
Postgresql connection parameters\&.

24
model/feed.go
View file

@ -7,7 +7,6 @@ package model // import "miniflux.app/model"
import (
"fmt"
"math"
"strings"
"time"
"miniflux.app/config"
@ -41,9 +40,10 @@ type Feed struct {
ReadCount int `json:"-"`
}
// List of supported schedulers.
const (
// SchedulerEntryCountBased represnets the name of the scheduler based on entry counts.
SchedulerEntryCountBased = "entry_count_based"
SchedulerRoundRobin = "round_robin"
SchedulerEntryFrequency = "entry_frequency"
)
func (f *Feed) String() string {
@ -102,24 +102,20 @@ func (f *Feed) CheckedNow() {
// ScheduleNextCheck set "next_check_at" of a feed based on the scheduler selected from the configuration.
func (f *Feed) ScheduleNextCheck(weeklyCount int) {
var nextCheckAt time.Time
switch strings.ToLower(config.Opts.PollingScheduler()) {
case SchedulerEntryCountBased:
switch config.Opts.PollingScheduler() {
case SchedulerEntryFrequency:
var intervalMinutes int
if weeklyCount == 0 {
intervalMinutes = config.Opts.SchedulerCountBasedMaxInterval()
intervalMinutes = config.Opts.SchedulerEntryFrequencyMaxInterval()
} else {
intervalMinutes = int(math.Round(float64(7*24*60) / float64(weeklyCount)))
}
intervalMinutes = int(math.Min(float64(intervalMinutes), float64(config.Opts.SchedulerCountBasedMaxInterval())))
intervalMinutes = int(math.Max(float64(intervalMinutes), float64(config.Opts.SchedulerCountBasedMinInterval())))
nextCheckAt = time.Now().Add(time.Minute * time.Duration(intervalMinutes))
intervalMinutes = int(math.Min(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMaxInterval())))
intervalMinutes = int(math.Max(float64(intervalMinutes), float64(config.Opts.SchedulerEntryFrequencyMinInterval())))
f.NextCheckAt = time.Now().Add(time.Minute * time.Duration(intervalMinutes))
default:
// round robin
// omit the interval because they are same for all feeds.
nextCheckAt = time.Now()
f.NextCheckAt = time.Now()
}
f.NextCheckAt = nextCheckAt
}
// Feeds is a list of feed

12
model/feed_test.go
View file

@ -133,9 +133,9 @@ func TestFeedScheduleNextCheckEntryCountBasedMaxInterval(t *testing.T) {
maxInterval := 5
minInterval := 1
os.Clearenv()
os.Setenv("POLLING_SCHEDULER", "entry_count_based")
os.Setenv("SCHEDULER_ENTRY_COUNT_BASED_MAX_INTERVAL", fmt.Sprintf("%d", maxInterval))
os.Setenv("SCHEDULER_ENTRY_COUNT_BASED_MIN_INTERVAL", fmt.Sprintf("%d", minInterval))
os.Setenv("POLLING_SCHEDULER", "entry_frequency")
os.Setenv("SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL", fmt.Sprintf("%d", maxInterval))
os.Setenv("SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL", fmt.Sprintf("%d", minInterval))
var err error
parser := config.NewParser()
@ -160,9 +160,9 @@ func TestFeedScheduleNextCheckEntryCountBasedMinInterval(t *testing.T) {
maxInterval := 500
minInterval := 100
os.Clearenv()
os.Setenv("POLLING_SCHEDULER", "entry_count_based")
os.Setenv("SCHEDULER_ENTRY_COUNT_BASED_MAX_INTERVAL", fmt.Sprintf("%d", maxInterval))
os.Setenv("SCHEDULER_ENTRY_COUNT_BASED_MIN_INTERVAL", fmt.Sprintf("%d", minInterval))
os.Setenv("POLLING_SCHEDULER", "entry_frequency")
os.Setenv("SCHEDULER_ENTRY_FREQUENCY_MAX_INTERVAL", fmt.Sprintf("%d", maxInterval))
os.Setenv("SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL", fmt.Sprintf("%d", minInterval))
var err error
parser := config.NewParser()

13
reader/feed/handler.go
View file

@ -8,6 +8,7 @@ import (
"fmt"
"time"
"miniflux.app/config"
"miniflux.app/errors"
"miniflux.app/http/client"
"miniflux.app/locale"
@ -90,13 +91,17 @@ func (h *Handler) RefreshFeed(userID, feedID int64) error {
return errors.NewLocalizedError(errNotFound, feedID)
}
weeklyCount, parametersErr := h.store.FeedSchedulerParameters(userID, feedID)
if parametersErr != nil {
return parametersErr
weeklyEntryCount := 0
if config.Opts.PollingScheduler() == model.SchedulerEntryFrequency {
var weeklyCountErr error
weeklyEntryCount, weeklyCountErr = h.store.WeeklyFeedEntryCount(userID, feedID)
if weeklyCountErr != nil {
return weeklyCountErr
}
}
originalFeed.CheckedNow()
originalFeed.ScheduleNextCheck(weeklyCount)
originalFeed.ScheduleNextCheck(weeklyEntryCount)
request := client.New(originalFeed.FeedURL)
request.WithCredentials(originalFeed.Username, originalFeed.Password)

19
storage/feed.go
View file

@ -8,9 +8,7 @@ import (
"database/sql"
"errors"
"fmt"
"strings"
"miniflux.app/config"
"miniflux.app/model"
"miniflux.app/timezone"
)
@ -274,14 +272,8 @@ func (s *Storage) fetchFeeds(feedQuery, counterQuery string, args ...interface{}
return feeds, nil
}
// FeedSchedulerParameters returns the parameters used for the scheduler.
func (s *Storage) FeedSchedulerParameters(userID, feedID int64) (int, error) {
scheduler := strings.ToLower(config.Opts.PollingScheduler())
if scheduler != model.SchedulerEntryCountBased {
return 0, nil
}
var weeklyCount int
// WeeklyFeedEntryCount returns the weekly entry count for a feed.
func (s *Storage) WeeklyFeedEntryCount(userID, feedID int64) (int, error) {
query := `
SELECT
count(*)
@ -293,15 +285,14 @@ func (s *Storage) FeedSchedulerParameters(userID, feedID int64) (int, error) {
entries.published_at BETWEEN (now() - interval '1 week') AND now();
`
err := s.db.QueryRow(query, userID, feedID).Scan(
&weeklyCount,
)
var weeklyCount int
err := s.db.QueryRow(query, userID, feedID).Scan(&weeklyCount)
switch {
case err == sql.ErrNoRows:
return 0, nil
case err != nil:
return 0, fmt.Errorf(`store: unable to fetch scheduler parameters for feed #%d: %v`, feedID, err)
return 0, fmt.Errorf(`store: unable to fetch weekly count for feed #%d: %v`, feedID, err)
}
return weeklyCount, nil