From 7c44238bae3aaee75fdcfe5b760f37bb0e86787c Mon Sep 17 00:00:00 2001
From: Shizun Ge <shizunge@gmail.com>
Date: Mon, 25 Jan 2021 23:41:36 -0600
Subject: [PATCH] Add global option POLLING_PARSING_ERROR_LIMIT
---
config/config_test.go | 18 ++++++++++++++++++
config/options.go | 9 +++++++++
config/parser.go | 2 ++
miniflux.1 | 5 +++++
storage/feed.go | 13 +++++++++++--
storage/job.go | 11 ++++++-----
6 files changed, 51 insertions(+), 7 deletions(-)
diff --git a/config/config_test.go b/config/config_test.go
index a59bf8a0..e5734e50 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -761,6 +761,24 @@ func TestDefautSchedulerCountBasedMinInterval(t *testing.T) {
}
}
+func TestPollingParsingErrorLimit(t *testing.T) {
+ os.Clearenv()
+ os.Setenv("POLLING_PARSING_ERROR_LIMIT", "100")
+
+ parser := NewParser()
+ opts, err := parser.ParseEnvironmentVariables()
+ if err != nil {
+ t.Fatalf(`Parsing failure: %v`, err)
+ }
+
+ expected := 100
+ result := opts.PollingParsingErrorLimit()
+
+ if result != expected {
+ t.Fatalf(`Unexpected POLLING_SCHEDULER value, got %v instead of %v`, result, expected)
+ }
+}
+
func TestOAuth2UserCreationWhenUnset(t *testing.T) {
os.Clearenv()
diff --git a/config/options.go b/config/options.go
index 81b62bc2..8448455b 100644
--- a/config/options.go
+++ b/config/options.go
@@ -29,6 +29,7 @@ const (
defaultPollingScheduler = "round_robin"
defaultSchedulerEntryFrequencyMinInterval = 5
defaultSchedulerEntryFrequencyMaxInterval = 24 * 60
+ defaultPollingParsingErrorLimit = 3
defaultRunMigrations = false
defaultDatabaseURL = "user=postgres password=postgres dbname=miniflux2 sslmode=disable"
defaultDatabaseMaxConns = 20
@@ -103,6 +104,7 @@ type Options struct {
pollingScheduler string
schedulerEntryFrequencyMinInterval int
schedulerEntryFrequencyMaxInterval int
+ pollingParsingErrorLimit int
workerPoolSize int
createAdmin bool
adminUsername string
@@ -159,6 +161,7 @@ func NewOptions() *Options {
pollingScheduler: defaultPollingScheduler,
schedulerEntryFrequencyMinInterval: defaultSchedulerEntryFrequencyMinInterval,
schedulerEntryFrequencyMaxInterval: defaultSchedulerEntryFrequencyMaxInterval,
+ pollingParsingErrorLimit: defaultPollingParsingErrorLimit,
workerPoolSize: defaultWorkerPoolSize,
createAdmin: defaultCreateAdmin,
proxyImages: defaultProxyImages,
@@ -318,6 +321,11 @@ func (o *Options) SchedulerEntryFrequencyMinInterval() int {
return o.schedulerEntryFrequencyMinInterval
}
+// PollingParsingErrorLimit returns the limit of errors when to stop polling.
+func (o *Options) PollingParsingErrorLimit() int {
+ return o.pollingParsingErrorLimit
+}
+
// IsOAuth2UserCreationAllowed returns true if user creation is allowed for OAuth2 users.
func (o *Options) IsOAuth2UserCreationAllowed() bool {
return o.oauth2UserCreationAllowed
@@ -493,6 +501,7 @@ func (o *Options) SortedOptions() []*Option {
"OAUTH2_USER_CREATION": o.oauth2UserCreationAllowed,
"POCKET_CONSUMER_KEY": o.pocketConsumerKey,
"POLLING_FREQUENCY": o.pollingFrequency,
+ "POLLING_PARSING_ERROR_LIMIT": o.pollingParsingErrorLimit,
"POLLING_SCHEDULER": o.pollingScheduler,
"PROXY_IMAGES": o.proxyImages,
"ROOT_URL": o.rootURL,
diff --git a/config/parser.go b/config/parser.go
index d075eddc..c7a422af 100644
--- a/config/parser.go
+++ b/config/parser.go
@@ -134,6 +134,8 @@ func (p *Parser) parseLines(lines []string) (err error) {
p.opts.schedulerEntryFrequencyMaxInterval = parseInt(value, defaultSchedulerEntryFrequencyMaxInterval)
case "SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL":
p.opts.schedulerEntryFrequencyMinInterval = parseInt(value, defaultSchedulerEntryFrequencyMinInterval)
+ case "POLLING_PARSING_ERROR_LIMIT":
+ p.opts.pollingParsingErrorLimit = parseInt(value, defaultPollingParsingErrorLimit)
case "PROXY_IMAGES":
p.opts.proxyImages = parseString(value, defaultProxyImages)
case "CREATE_ADMIN":
diff --git a/miniflux.1 b/miniflux.1
index 69293b1c..303d3c81 100644
--- a/miniflux.1
+++ b/miniflux.1
@@ -146,6 +146,11 @@ Minimum interval in minutes for the entry frequency scheduler\&.
.br
Default is 5 minutes\&.
.TP
+.B POLLING_PARSING_ERROR_LIMIT
+The maximum number of parsing errors that the program will try before stopping polling a feed. Once the limit is reached, the user must refresh the feed manually. Set to 0 for unlimited.
+.br
+Default is 3\&.
+.TP
.B DATABASE_URL
Postgresql connection parameters\&.
.br
diff --git a/storage/feed.go b/storage/feed.go
index c4524384..12e066e7 100644
--- a/storage/feed.go
+++ b/storage/feed.go
@@ -9,6 +9,7 @@ import (
"errors"
"fmt"
+ "miniflux.app/config"
"miniflux.app/model"
)
@@ -80,9 +81,13 @@ func (s *Storage) CountFeeds(userID int64) int {
// CountUserFeedsWithErrors returns the number of feeds with parsing errors that belong to the given user.
func (s *Storage) CountUserFeedsWithErrors(userID int64) int {
+ pollingParsingErrorLimit := config.Opts.PollingParsingErrorLimit()
+ if pollingParsingErrorLimit <= 0 {
+ pollingParsingErrorLimit = 1
+ }
query := `SELECT count(*) FROM feeds WHERE user_id=$1 AND parsing_error_count >= $2`
var result int
- err := s.db.QueryRow(query, userID, maxParsingError).Scan(&result)
+ err := s.db.QueryRow(query, userID, pollingParsingErrorLimit).Scan(&result)
if err != nil {
return 0
}
@@ -92,9 +97,13 @@ func (s *Storage) CountUserFeedsWithErrors(userID int64) int {
// CountAllFeedsWithErrors returns the number of feeds with parsing errors.
func (s *Storage) CountAllFeedsWithErrors() int {
+ pollingParsingErrorLimit := config.Opts.PollingParsingErrorLimit()
+ if pollingParsingErrorLimit <= 0 {
+ pollingParsingErrorLimit = 1
+ }
query := `SELECT count(*) FROM feeds WHERE parsing_error_count >= $1`
var result int
- err := s.db.QueryRow(query, maxParsingError).Scan(&result)
+ err := s.db.QueryRow(query, pollingParsingErrorLimit).Scan(&result)
if err != nil {
return 0
}
diff --git a/storage/job.go b/storage/job.go
index b1b28044..5b10468a 100644
--- a/storage/job.go
+++ b/storage/job.go
@@ -7,13 +7,13 @@ package storage // import "miniflux.app/storage"
import (
"fmt"
+ "miniflux.app/config"
"miniflux.app/model"
)
-const maxParsingError = 3
-
// NewBatch returns a serie of jobs.
func (s *Storage) NewBatch(batchSize int) (jobs model.JobList, err error) {
+ pollingParsingErrorLimit := config.Opts.PollingParsingErrorLimit()
query := `
SELECT
id,
@@ -21,10 +21,11 @@ func (s *Storage) NewBatch(batchSize int) (jobs model.JobList, err error) {
FROM
feeds
WHERE
- parsing_error_count < $1 AND disabled is false AND next_check_at < now()
- ORDER BY next_check_at ASC LIMIT %d
+ disabled is false AND next_check_at < now() AND
+ CASE WHEN $1 > 0 THEN parsing_error_count < $1 ELSE parsing_error_count >= 0 END
+ ORDER BY next_check_at ASC LIMIT $2
`
- return s.fetchBatchRows(fmt.Sprintf(query, batchSize), maxParsingError)
+ return s.fetchBatchRows(query, pollingParsingErrorLimit, batchSize)
}
// NewUserBatch returns a serie of jobs but only for a given user.