diff --git a/config/config_test.go b/config/config_test.go index a59bf8a0..e5734e50 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -761,6 +761,24 @@ func TestDefautSchedulerCountBasedMinInterval(t *testing.T) { } } +func TestPollingParsingErrorLimit(t *testing.T) { + os.Clearenv() + os.Setenv("POLLING_PARSING_ERROR_LIMIT", "100") + + parser := NewParser() + opts, err := parser.ParseEnvironmentVariables() + if err != nil { + t.Fatalf(`Parsing failure: %v`, err) + } + + expected := 100 + result := opts.PollingParsingErrorLimit() + + if result != expected { + t.Fatalf(`Unexpected POLLING_SCHEDULER value, got %v instead of %v`, result, expected) + } +} + func TestOAuth2UserCreationWhenUnset(t *testing.T) { os.Clearenv() diff --git a/config/options.go b/config/options.go index 81b62bc2..8448455b 100644 --- a/config/options.go +++ b/config/options.go @@ -29,6 +29,7 @@ const ( defaultPollingScheduler = "round_robin" defaultSchedulerEntryFrequencyMinInterval = 5 defaultSchedulerEntryFrequencyMaxInterval = 24 * 60 + defaultPollingParsingErrorLimit = 3 defaultRunMigrations = false defaultDatabaseURL = "user=postgres password=postgres dbname=miniflux2 sslmode=disable" defaultDatabaseMaxConns = 20 @@ -103,6 +104,7 @@ type Options struct { pollingScheduler string schedulerEntryFrequencyMinInterval int schedulerEntryFrequencyMaxInterval int + pollingParsingErrorLimit int workerPoolSize int createAdmin bool adminUsername string @@ -159,6 +161,7 @@ func NewOptions() *Options { pollingScheduler: defaultPollingScheduler, schedulerEntryFrequencyMinInterval: defaultSchedulerEntryFrequencyMinInterval, schedulerEntryFrequencyMaxInterval: defaultSchedulerEntryFrequencyMaxInterval, + pollingParsingErrorLimit: defaultPollingParsingErrorLimit, workerPoolSize: defaultWorkerPoolSize, createAdmin: defaultCreateAdmin, proxyImages: defaultProxyImages, @@ -318,6 +321,11 @@ func (o *Options) SchedulerEntryFrequencyMinInterval() int { return o.schedulerEntryFrequencyMinInterval } +// PollingParsingErrorLimit returns the limit of errors when to stop polling. +func (o *Options) PollingParsingErrorLimit() int { + return o.pollingParsingErrorLimit +} + // IsOAuth2UserCreationAllowed returns true if user creation is allowed for OAuth2 users. func (o *Options) IsOAuth2UserCreationAllowed() bool { return o.oauth2UserCreationAllowed @@ -493,6 +501,7 @@ func (o *Options) SortedOptions() []*Option { "OAUTH2_USER_CREATION": o.oauth2UserCreationAllowed, "POCKET_CONSUMER_KEY": o.pocketConsumerKey, "POLLING_FREQUENCY": o.pollingFrequency, + "POLLING_PARSING_ERROR_LIMIT": o.pollingParsingErrorLimit, "POLLING_SCHEDULER": o.pollingScheduler, "PROXY_IMAGES": o.proxyImages, "ROOT_URL": o.rootURL, diff --git a/config/parser.go b/config/parser.go index d075eddc..c7a422af 100644 --- a/config/parser.go +++ b/config/parser.go @@ -134,6 +134,8 @@ func (p *Parser) parseLines(lines []string) (err error) { p.opts.schedulerEntryFrequencyMaxInterval = parseInt(value, defaultSchedulerEntryFrequencyMaxInterval) case "SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL": p.opts.schedulerEntryFrequencyMinInterval = parseInt(value, defaultSchedulerEntryFrequencyMinInterval) + case "POLLING_PARSING_ERROR_LIMIT": + p.opts.pollingParsingErrorLimit = parseInt(value, defaultPollingParsingErrorLimit) case "PROXY_IMAGES": p.opts.proxyImages = parseString(value, defaultProxyImages) case "CREATE_ADMIN": diff --git a/miniflux.1 b/miniflux.1 index 69293b1c..303d3c81 100644 --- a/miniflux.1 +++ b/miniflux.1 @@ -146,6 +146,11 @@ Minimum interval in minutes for the entry frequency scheduler\&. .br Default is 5 minutes\&. .TP +.B POLLING_PARSING_ERROR_LIMIT +The maximum number of parsing errors that the program will try before stopping polling a feed. Once the limit is reached, the user must refresh the feed manually. Set to 0 for unlimited. +.br +Default is 3\&. +.TP .B DATABASE_URL Postgresql connection parameters\&. .br diff --git a/storage/feed.go b/storage/feed.go index c4524384..12e066e7 100644 --- a/storage/feed.go +++ b/storage/feed.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" + "miniflux.app/config" "miniflux.app/model" ) @@ -80,9 +81,13 @@ func (s *Storage) CountFeeds(userID int64) int { // CountUserFeedsWithErrors returns the number of feeds with parsing errors that belong to the given user. func (s *Storage) CountUserFeedsWithErrors(userID int64) int { + pollingParsingErrorLimit := config.Opts.PollingParsingErrorLimit() + if pollingParsingErrorLimit <= 0 { + pollingParsingErrorLimit = 1 + } query := `SELECT count(*) FROM feeds WHERE user_id=$1 AND parsing_error_count >= $2` var result int - err := s.db.QueryRow(query, userID, maxParsingError).Scan(&result) + err := s.db.QueryRow(query, userID, pollingParsingErrorLimit).Scan(&result) if err != nil { return 0 } @@ -92,9 +97,13 @@ func (s *Storage) CountUserFeedsWithErrors(userID int64) int { // CountAllFeedsWithErrors returns the number of feeds with parsing errors. func (s *Storage) CountAllFeedsWithErrors() int { + pollingParsingErrorLimit := config.Opts.PollingParsingErrorLimit() + if pollingParsingErrorLimit <= 0 { + pollingParsingErrorLimit = 1 + } query := `SELECT count(*) FROM feeds WHERE parsing_error_count >= $1` var result int - err := s.db.QueryRow(query, maxParsingError).Scan(&result) + err := s.db.QueryRow(query, pollingParsingErrorLimit).Scan(&result) if err != nil { return 0 } diff --git a/storage/job.go b/storage/job.go index b1b28044..5b10468a 100644 --- a/storage/job.go +++ b/storage/job.go @@ -7,13 +7,13 @@ package storage // import "miniflux.app/storage" import ( "fmt" + "miniflux.app/config" "miniflux.app/model" ) -const maxParsingError = 3 - // NewBatch returns a serie of jobs. func (s *Storage) NewBatch(batchSize int) (jobs model.JobList, err error) { + pollingParsingErrorLimit := config.Opts.PollingParsingErrorLimit() query := ` SELECT id, @@ -21,10 +21,11 @@ func (s *Storage) NewBatch(batchSize int) (jobs model.JobList, err error) { FROM feeds WHERE - parsing_error_count < $1 AND disabled is false AND next_check_at < now() - ORDER BY next_check_at ASC LIMIT %d + disabled is false AND next_check_at < now() AND + CASE WHEN $1 > 0 THEN parsing_error_count < $1 ELSE parsing_error_count >= 0 END + ORDER BY next_check_at ASC LIMIT $2 ` - return s.fetchBatchRows(fmt.Sprintf(query, batchSize), maxParsingError) + return s.fetchBatchRows(query, pollingParsingErrorLimit, batchSize) } // NewUserBatch returns a serie of jobs but only for a given user.