Refactor Batch Builder and prevent accidental and excessive refreshes from the web ui
This commit is contained in:
Frédéric Guillot
parent
95ee1c423b
commit
4cc99881d8
32 changed files with 251 additions and 176 deletions
|
|
@ -9,6 +9,7 @@ import (
|
|||
"net/http"
|
||||
"time"
|
||||
|
||||
"miniflux.app/v2/internal/config"
|
||||
"miniflux.app/v2/internal/http/request"
|
||||
"miniflux.app/v2/internal/http/response/json"
|
||||
"miniflux.app/v2/internal/model"
|
||||
|
|
@ -136,7 +137,14 @@ func (h *handler) refreshCategory(w http.ResponseWriter, r *http.Request) {
|
|||
userID := request.UserID(r)
|
||||
categoryID := request.RouteInt64Param(r, "categoryID")
|
||||
|
||||
jobs, err := h.store.NewCategoryBatch(userID, categoryID, h.store.CountFeeds(userID))
|
||||
batchBuilder := h.store.NewBatchBuilder()
|
||||
batchBuilder.WithErrorLimit(config.Opts.PollingParsingErrorLimit())
|
||||
batchBuilder.WithoutDisabledFeeds()
|
||||
batchBuilder.WithUserID(userID)
|
||||
batchBuilder.WithCategoryID(categoryID)
|
||||
batchBuilder.WithNextCheckExpired()
|
||||
|
||||
jobs, err := batchBuilder.FetchJobs()
|
||||
if err != nil {
|
||||
json.ServerError(w, r, err)
|
||||
return
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import (
|
|||
"net/http"
|
||||
"time"
|
||||
|
||||
"miniflux.app/v2/internal/config"
|
||||
"miniflux.app/v2/internal/http/request"
|
||||
"miniflux.app/v2/internal/http/response/json"
|
||||
"miniflux.app/v2/internal/model"
|
||||
|
|
@ -69,7 +70,14 @@ func (h *handler) refreshFeed(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
func (h *handler) refreshAllFeeds(w http.ResponseWriter, r *http.Request) {
|
||||
userID := request.UserID(r)
|
||||
jobs, err := h.store.NewUserBatch(userID, h.store.CountFeeds(userID))
|
||||
|
||||
batchBuilder := h.store.NewBatchBuilder()
|
||||
batchBuilder.WithErrorLimit(config.Opts.PollingParsingErrorLimit())
|
||||
batchBuilder.WithoutDisabledFeeds()
|
||||
batchBuilder.WithNextCheckExpired()
|
||||
batchBuilder.WithUserID(userID)
|
||||
|
||||
jobs, err := batchBuilder.FetchJobs()
|
||||
if err != nil {
|
||||
json.ServerError(w, r, err)
|
||||
return
|
||||
|
|
|
|||
|
|
@ -18,7 +18,15 @@ func refreshFeeds(store *storage.Storage) {
|
|||
var wg sync.WaitGroup
|
||||
|
||||
startTime := time.Now()
|
||||
jobs, err := store.NewBatch(config.Opts.BatchSize())
|
||||
|
||||
// Generate a batch of feeds for any user that has feeds to refresh.
|
||||
batchBuilder := store.NewBatchBuilder()
|
||||
batchBuilder.WithBatchSize(config.Opts.BatchSize())
|
||||
batchBuilder.WithErrorLimit(config.Opts.PollingParsingErrorLimit())
|
||||
batchBuilder.WithoutDisabledFeeds()
|
||||
batchBuilder.WithNextCheckExpired()
|
||||
|
||||
jobs, err := batchBuilder.FetchJobs()
|
||||
if err != nil {
|
||||
slog.Error("Unable to fetch jobs from database", slog.Any("error", err))
|
||||
return
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ func runScheduler(store *storage.Storage, pool *worker.Pool) {
|
|||
pool,
|
||||
config.Opts.PollingFrequency(),
|
||||
config.Opts.BatchSize(),
|
||||
config.Opts.PollingParsingErrorLimit(),
|
||||
)
|
||||
|
||||
go cleanupScheduler(
|
||||
|
|
@ -28,10 +29,16 @@ func runScheduler(store *storage.Storage, pool *worker.Pool) {
|
|||
)
|
||||
}
|
||||
|
||||
func feedScheduler(store *storage.Storage, pool *worker.Pool, frequency, batchSize int) {
|
||||
func feedScheduler(store *storage.Storage, pool *worker.Pool, frequency, batchSize, errorLimit int) {
|
||||
for range time.Tick(time.Duration(frequency) * time.Minute) {
|
||||
jobs, err := store.NewBatch(batchSize)
|
||||
if err != nil {
|
||||
// Generate a batch of feeds for any user that has feeds to refresh.
|
||||
batchBuilder := store.NewBatchBuilder()
|
||||
batchBuilder.WithBatchSize(batchSize)
|
||||
batchBuilder.WithErrorLimit(errorLimit)
|
||||
batchBuilder.WithoutDisabledFeeds()
|
||||
batchBuilder.WithNextCheckExpired()
|
||||
|
||||
if jobs, err := batchBuilder.FetchJobs(); err != nil {
|
||||
slog.Error("Unable to fetch jobs from database", slog.Any("error", err))
|
||||
} else {
|
||||
slog.Info("Created a batch of feeds",
|
||||
|
|
|
|||
|
|
@ -3,7 +3,10 @@
|
|||
|
||||
package request // import "miniflux.app/v2/internal/http/request"
|
||||
|
||||
import "net/http"
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// ContextKey represents a context key.
|
||||
type ContextKey int
|
||||
|
|
@ -24,6 +27,7 @@ const (
|
|||
FlashMessageContextKey
|
||||
FlashErrorMessageContextKey
|
||||
PocketRequestTokenContextKey
|
||||
LastForceRefreshContextKey
|
||||
ClientIPContextKey
|
||||
GoogleReaderToken
|
||||
)
|
||||
|
|
@ -114,6 +118,16 @@ func PocketRequestToken(r *http.Request) string {
|
|||
return getContextStringValue(r, PocketRequestTokenContextKey)
|
||||
}
|
||||
|
||||
// LastForceRefresh returns the last force refresh timestamp.
|
||||
func LastForceRefresh(r *http.Request) int64 {
|
||||
jsonStringValue := getContextStringValue(r, LastForceRefreshContextKey)
|
||||
timestamp, err := strconv.ParseInt(jsonStringValue, 10, 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return timestamp
|
||||
}
|
||||
|
||||
// ClientIP returns the client IP address stored in the context.
|
||||
func ClientIP(r *http.Request) string {
|
||||
return getContextStringValue(r, ClientIPContextKey)
|
||||
|
|
|
|||
|
|
@ -453,6 +453,6 @@
|
|||
"You are not authorized to access this resource (invalid username/password)": "Sie sind nicht berechtigt, auf diese Ressource zuzugreifen (Benutzername/Passwort ungültig)",
|
||||
"Unable to fetch this resource (Status Code = %d)": "Ressource konnte nicht abgerufen werden (code=%d)",
|
||||
"Resource not found (404), this feed doesn't exist anymore, check the feed URL": "Ressource nicht gefunden (404), dieses Abonnement existiert nicht mehr, überprüfen Sie die Abonnement-URL",
|
||||
"page.background_feed_refresh.title": "Hintergrundaktualisierung",
|
||||
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -434,6 +434,6 @@
|
|||
"πριν %d έτος",
|
||||
"πριν %d έτη"
|
||||
],
|
||||
"page.background_feed_refresh.title": "Background feed refresh",
|
||||
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -434,6 +434,6 @@
|
|||
"%d year ago",
|
||||
"%d years ago"
|
||||
],
|
||||
"page.background_feed_refresh.title": "Background feed refresh",
|
||||
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -434,6 +434,6 @@
|
|||
"hace %d año",
|
||||
"hace %d años"
|
||||
],
|
||||
"page.background_feed_refresh.title": "Background feed refresh",
|
||||
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -434,6 +434,6 @@
|
|||
"%d vuosi sitten",
|
||||
"%d vuotta sitten"
|
||||
],
|
||||
"page.background_feed_refresh.title": "Background feed refresh",
|
||||
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -453,6 +453,6 @@
|
|||
"You are not authorized to access this resource (invalid username/password)": "Vous n'êtes pas autorisé à accéder à cette ressource (nom d'utilisateur / mot de passe incorrect)",
|
||||
"Unable to fetch this resource (Status Code = %d)": "Impossible de récupérer cette ressource (code=%d)",
|
||||
"Resource not found (404), this feed doesn't exist anymore, check the feed URL": "Page introuvable (404), cet abonnement n'existe plus, vérifiez l'adresse du flux",
|
||||
"page.background_feed_refresh.title": "Actualisation des abonnements en arrière-plan",
|
||||
"alert.too_many_feeds_refresh": "Vous avez déclenché trop d'actualisations de flux. Veuillez attendre 30 minutes avant de réessayer.",
|
||||
"alert.background_feed_refresh": "Les abonnements sont en cours d'actualisation en arrière-plan. Vous pouvez continuer à naviguer dans l'application."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -434,6 +434,6 @@
|
|||
"%d साल पहले",
|
||||
"%d वर्षों पहले"
|
||||
],
|
||||
"page.background_feed_refresh.title": "फ़ीड रीफ़्रेश किया जा रहा है",
|
||||
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -444,6 +444,6 @@
|
|||
"You are not authorized to access this resource (invalid username/password)": "Anda tidak memiliki izin yang cukup untuk mengakses umpan ini (nama pengguna/kata sandi tidak valid)",
|
||||
"Unable to fetch this resource (Status Code = %d)": "Tidak bisa mengambil umpan ini (Kode Status = %d)",
|
||||
"Resource not found (404), this feed doesn't exist anymore, check the feed URL": "Umpan tidak ditemukan (404), umpan ini tidak ada lagi, periksa URL umpan",
|
||||
"page.background_feed_refresh.title": "Memuat ulang umpan di latar belakang",
|
||||
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -434,6 +434,6 @@
|
|||
"%d anno fa",
|
||||
"%d anni fa"
|
||||
],
|
||||
"page.background_feed_refresh.title": "Aggiornamento in background",
|
||||
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -434,6 +434,6 @@
|
|||
"%d 年前",
|
||||
"%d 年前"
|
||||
],
|
||||
"page.background_feed_refresh.title": "バックグラウンドでフィードを更新中",
|
||||
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -451,6 +451,6 @@
|
|||
"Invalid SSL certificate (original error: %q)": "Ongeldig SSL-certificaat (originele error: %q)",
|
||||
"This website is unreachable (original error: %q)": "Deze website is onbereikbaar (originele error: %q)",
|
||||
"Website unreachable, the request timed out after %d seconds": "Website onbereikbaar, de request gaf een timeout na %d seconden",
|
||||
"page.background_feed_refresh.title": "Achtergrond vernieuwen",
|
||||
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -459,6 +459,6 @@
|
|||
"Invalid SSL certificate (original error: %q)": "Certyfikat SSL jest nieprawidłowy (błąd: %q)",
|
||||
"This website is unreachable (original error: %q)": "Ta strona jest niedostępna (błąd: %q)",
|
||||
"Website unreachable, the request timed out after %d seconds": "Strona internetowa nieosiągalna, żądanie wygasło po %d sekundach",
|
||||
"page.background_feed_refresh.title": "Odświeżanie kanałów w tle",
|
||||
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -434,6 +434,6 @@
|
|||
"há %d ano",
|
||||
"há %d anos"
|
||||
],
|
||||
"page.background_feed_refresh.title": "Atualização de fonte em segundo plano",
|
||||
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -442,6 +442,6 @@
|
|||
"%d года назад",
|
||||
"%d лет назад"
|
||||
],
|
||||
"page.background_feed_refresh.title": "Обновление подписок",
|
||||
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -434,6 +434,6 @@
|
|||
"%d yıl önce",
|
||||
"%d yıl önce"
|
||||
],
|
||||
"page.background_feed_refresh.title": "Arka plan beslemesi yenileme",
|
||||
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -443,6 +443,6 @@
|
|||
"%d роки тому",
|
||||
"%d років тому"
|
||||
],
|
||||
"page.background_feed_refresh.title": "Оновлення стрічок в фоновому режимі",
|
||||
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -444,6 +444,6 @@
|
|||
"Invalid SSL certificate (original error: %q)": "无效的 SSL 证书 (原始错误: %q)",
|
||||
"This website is unreachable (original error: %q)": "该网站永久不可达 (原始错误: %q)",
|
||||
"Website unreachable, the request timed out after %d seconds": "网站不可达, 请求已在 %d 秒后超时",
|
||||
"page.background_feed_refresh.title": "后台更新",
|
||||
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -452,6 +452,6 @@
|
|||
"Invalid SSL certificate (original error: %q)": "無效的 SSL 憑證 (錯誤: %q)",
|
||||
"This website is unreachable (original error: %q)": "該網站永久無法訪問(原始錯誤: %q)",
|
||||
"Website unreachable, the request timed out after %d seconds": "網站無法訪問, 請求已在 %d 秒後超時",
|
||||
"page.background_feed_refresh.title": "背景更新",
|
||||
"alert.too_many_feeds_refresh": "You have triggered too many feed refreshes. Please wait 30 minutes before trying again.",
|
||||
"alert.background_feed_refresh": "All feeds are being refreshed in the background. You can continue to use Miniflux while this process is running."
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,11 +20,21 @@ type SessionData struct {
|
|||
Language string `json:"language"`
|
||||
Theme string `json:"theme"`
|
||||
PocketRequestToken string `json:"pocket_request_token"`
|
||||
LastForceRefresh string `json:"last_force_refresh"`
|
||||
}
|
||||
|
||||
func (s SessionData) String() string {
|
||||
return fmt.Sprintf(`CSRF=%q, OAuth2State=%q, OAuth2CodeVerifier=%q, FlashMsg=%q, FlashErrMsg=%q, Lang=%q, Theme=%q, PocketTkn=%q`,
|
||||
s.CSRF, s.OAuth2State, s.OAuth2CodeVerifier, s.FlashMessage, s.FlashErrorMessage, s.Language, s.Theme, s.PocketRequestToken)
|
||||
return fmt.Sprintf(`CSRF=%q, OAuth2State=%q, OAuth2CodeVerifier=%q, FlashMsg=%q, FlashErrMsg=%q, Lang=%q, Theme=%q, PocketTkn=%q, LastForceRefresh=%s`,
|
||||
s.CSRF,
|
||||
s.OAuth2State,
|
||||
s.OAuth2CodeVerifier,
|
||||
s.FlashMessage,
|
||||
s.FlashErrorMessage,
|
||||
s.Language,
|
||||
s.Theme,
|
||||
s.PocketRequestToken,
|
||||
s.LastForceRefresh,
|
||||
)
|
||||
}
|
||||
|
||||
// Value converts the session data to JSON.
|
||||
|
|
|
|||
91
internal/storage/batch.go
Normal file
91
internal/storage/batch.go
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package storage // import "miniflux.app/v2/internal/storage"
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"miniflux.app/v2/internal/model"
|
||||
)
|
||||
|
||||
type BatchBuilder struct {
|
||||
db *sql.DB
|
||||
args []any
|
||||
conditions []string
|
||||
limit int
|
||||
}
|
||||
|
||||
func (s *Storage) NewBatchBuilder() *BatchBuilder {
|
||||
return &BatchBuilder{
|
||||
db: s.db,
|
||||
}
|
||||
}
|
||||
|
||||
func (b *BatchBuilder) WithBatchSize(batchSize int) *BatchBuilder {
|
||||
b.limit = batchSize
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *BatchBuilder) WithUserID(userID int64) *BatchBuilder {
|
||||
b.conditions = append(b.conditions, fmt.Sprintf("user_id = $%d", len(b.args)+1))
|
||||
b.args = append(b.args, userID)
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *BatchBuilder) WithCategoryID(categoryID int64) *BatchBuilder {
|
||||
b.conditions = append(b.conditions, fmt.Sprintf("category_id = $%d", len(b.args)+1))
|
||||
b.args = append(b.args, categoryID)
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *BatchBuilder) WithErrorLimit(limit int) *BatchBuilder {
|
||||
if limit > 0 {
|
||||
b.conditions = append(b.conditions, fmt.Sprintf("parsing_error_count < $%d", len(b.args)+1))
|
||||
b.args = append(b.args, limit)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *BatchBuilder) WithNextCheckExpired() *BatchBuilder {
|
||||
b.conditions = append(b.conditions, "next_check_at < now()")
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *BatchBuilder) WithoutDisabledFeeds() *BatchBuilder {
|
||||
b.conditions = append(b.conditions, "disabled is false")
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *BatchBuilder) FetchJobs() (jobs model.JobList, err error) {
|
||||
var parts []string
|
||||
parts = append(parts, `SELECT id, user_id FROM feeds`)
|
||||
|
||||
if len(b.conditions) > 0 {
|
||||
parts = append(parts, fmt.Sprintf("WHERE %s", strings.Join(b.conditions, " AND ")))
|
||||
}
|
||||
|
||||
if b.limit > 0 {
|
||||
parts = append(parts, fmt.Sprintf("ORDER BY next_check_at ASC LIMIT %d", b.limit))
|
||||
}
|
||||
|
||||
query := strings.Join(parts, " ")
|
||||
rows, err := b.db.Query(query, b.args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(`store: unable to fetch batch of jobs: %v`, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var job model.Job
|
||||
if err := rows.Scan(&job.FeedID, &job.UserID); err != nil {
|
||||
return nil, fmt.Errorf(`store: unable to fetch job: %v`, err)
|
||||
}
|
||||
|
||||
jobs = append(jobs, job)
|
||||
}
|
||||
|
||||
return jobs, nil
|
||||
}
|
||||
|
|
@ -87,17 +87,6 @@ func (s *Storage) CountAllFeeds() map[string]int64 {
|
|||
return results
|
||||
}
|
||||
|
||||
// CountFeeds returns the number of feeds that belongs to the given user.
|
||||
func (s *Storage) CountFeeds(userID int64) int {
|
||||
var result int
|
||||
err := s.db.QueryRow(`SELECT count(*) FROM feeds WHERE user_id=$1`, userID).Scan(&result)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// CountUserFeedsWithErrors returns the number of feeds with parsing errors that belong to the given user.
|
||||
func (s *Storage) CountUserFeedsWithErrors(userID int64) int {
|
||||
pollingParsingErrorLimit := config.Opts.PollingParsingErrorLimit()
|
||||
|
|
|
|||
|
|
@ -1,81 +0,0 @@
|
|||
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package storage // import "miniflux.app/v2/internal/storage"
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"miniflux.app/v2/internal/config"
|
||||
"miniflux.app/v2/internal/model"
|
||||
)
|
||||
|
||||
// NewBatch returns a series of jobs.
|
||||
func (s *Storage) NewBatch(batchSize int) (jobs model.JobList, err error) {
|
||||
pollingParsingErrorLimit := config.Opts.PollingParsingErrorLimit()
|
||||
query := `
|
||||
SELECT
|
||||
id,
|
||||
user_id
|
||||
FROM
|
||||
feeds
|
||||
WHERE
|
||||
disabled is false AND next_check_at < now() AND
|
||||
CASE WHEN $1 > 0 THEN parsing_error_count < $1 ELSE parsing_error_count >= 0 END
|
||||
ORDER BY next_check_at ASC LIMIT $2
|
||||
`
|
||||
return s.fetchBatchRows(query, pollingParsingErrorLimit, batchSize)
|
||||
}
|
||||
|
||||
// NewUserBatch returns a series of jobs but only for a given user.
|
||||
func (s *Storage) NewUserBatch(userID int64, batchSize int) (jobs model.JobList, err error) {
|
||||
// We do not take the error counter into consideration when the given
|
||||
// user refresh manually all his feeds to force a refresh.
|
||||
query := `
|
||||
SELECT
|
||||
id,
|
||||
user_id
|
||||
FROM
|
||||
feeds
|
||||
WHERE
|
||||
user_id=$1 AND disabled is false AND next_check_at < now()
|
||||
ORDER BY next_check_at ASC LIMIT %d
|
||||
`
|
||||
return s.fetchBatchRows(fmt.Sprintf(query, batchSize), userID)
|
||||
}
|
||||
|
||||
// NewCategoryBatch returns a series of jobs but only for a given category.
|
||||
func (s *Storage) NewCategoryBatch(userID int64, categoryID int64, batchSize int) (jobs model.JobList, err error) {
|
||||
// We do not take the error counter into consideration when the given
|
||||
// user refresh manually all his feeds to force a refresh.
|
||||
query := `
|
||||
SELECT
|
||||
id,
|
||||
user_id
|
||||
FROM
|
||||
feeds
|
||||
WHERE
|
||||
user_id=$1 AND category_id=$2 AND disabled is false AND next_check_at < now()
|
||||
ORDER BY next_check_at ASC LIMIT %d
|
||||
`
|
||||
return s.fetchBatchRows(fmt.Sprintf(query, batchSize), userID, categoryID)
|
||||
}
|
||||
|
||||
func (s *Storage) fetchBatchRows(query string, args ...interface{}) (jobs model.JobList, err error) {
|
||||
rows, err := s.db.Query(query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf(`store: unable to fetch batch of jobs: %v`, err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var job model.Job
|
||||
if err := rows.Scan(&job.FeedID, &job.UserID); err != nil {
|
||||
return nil, fmt.Errorf(`store: unable to fetch job: %v`, err)
|
||||
}
|
||||
|
||||
jobs = append(jobs, job)
|
||||
}
|
||||
|
||||
return jobs, nil
|
||||
}
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
{{ define "title"}}{{ t "page.background_feed_refresh.title" }}{{ end }}
|
||||
|
||||
{{ define "content"}}
|
||||
<section class="page-header">
|
||||
<h1>{{ t "page.background_feed_refresh.title" }}</h1>
|
||||
{{ template "feed_menu" }}
|
||||
</section>
|
||||
|
||||
<p class="alert alert-success">{{ t "alert.background_feed_refresh" }}</p>
|
||||
|
||||
{{ end }}
|
||||
|
|
@ -6,10 +6,13 @@ package ui // import "miniflux.app/v2/internal/ui"
|
|||
import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"miniflux.app/v2/internal/http/request"
|
||||
"miniflux.app/v2/internal/http/response/html"
|
||||
"miniflux.app/v2/internal/http/route"
|
||||
"miniflux.app/v2/internal/locale"
|
||||
"miniflux.app/v2/internal/ui/session"
|
||||
)
|
||||
|
||||
func (h *handler) refreshCategoryEntriesPage(w http.ResponseWriter, r *http.Request) {
|
||||
|
|
@ -25,8 +28,21 @@ func (h *handler) refreshCategoryFeedsPage(w http.ResponseWriter, r *http.Reques
|
|||
func (h *handler) refreshCategory(w http.ResponseWriter, r *http.Request) int64 {
|
||||
userID := request.UserID(r)
|
||||
categoryID := request.RouteInt64Param(r, "categoryID")
|
||||
printer := locale.NewPrinter(request.UserLanguage(r))
|
||||
sess := session.New(h.store, request.SessionID(r))
|
||||
|
||||
jobs, err := h.store.NewCategoryBatch(userID, categoryID, h.store.CountFeeds(userID))
|
||||
// Avoid accidental and excessive refreshes.
|
||||
if time.Now().UTC().Unix()-request.LastForceRefresh(r) < 1800 {
|
||||
sess.NewFlashErrorMessage(printer.Printf("alert.too_many_feeds_refresh"))
|
||||
} else {
|
||||
// We allow the end-user to force refresh all its feeds in this category
|
||||
// without taking into consideration the number of errors.
|
||||
batchBuilder := h.store.NewBatchBuilder()
|
||||
batchBuilder.WithoutDisabledFeeds()
|
||||
batchBuilder.WithUserID(userID)
|
||||
batchBuilder.WithCategoryID(categoryID)
|
||||
|
||||
jobs, err := batchBuilder.FetchJobs()
|
||||
if err != nil {
|
||||
html.ServerError(w, r, err)
|
||||
return 0
|
||||
|
|
@ -41,5 +57,9 @@ func (h *handler) refreshCategory(w http.ResponseWriter, r *http.Request) int64
|
|||
|
||||
go h.pool.Push(jobs)
|
||||
|
||||
sess.SetLastForceRefresh()
|
||||
sess.NewFlashMessage(printer.Printf("alert.background_feed_refresh"))
|
||||
}
|
||||
|
||||
return categoryID
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,13 +6,14 @@ package ui // import "miniflux.app/v2/internal/ui"
|
|||
import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"miniflux.app/v2/internal/http/request"
|
||||
"miniflux.app/v2/internal/http/response/html"
|
||||
"miniflux.app/v2/internal/http/route"
|
||||
"miniflux.app/v2/internal/locale"
|
||||
feedHandler "miniflux.app/v2/internal/reader/handler"
|
||||
"miniflux.app/v2/internal/ui/session"
|
||||
"miniflux.app/v2/internal/ui/view"
|
||||
)
|
||||
|
||||
func (h *handler) refreshFeed(w http.ResponseWriter, r *http.Request) {
|
||||
|
|
@ -32,14 +33,20 @@ func (h *handler) refreshFeed(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
func (h *handler) refreshAllFeeds(w http.ResponseWriter, r *http.Request) {
|
||||
userID := request.UserID(r)
|
||||
printer := locale.NewPrinter(request.UserLanguage(r))
|
||||
sess := session.New(h.store, request.SessionID(r))
|
||||
|
||||
user, err := h.store.UserByID(userID)
|
||||
if err != nil {
|
||||
html.ServerError(w, r, err)
|
||||
return
|
||||
}
|
||||
// Avoid accidental and excessive refreshes.
|
||||
if time.Now().UTC().Unix()-request.LastForceRefresh(r) < 1800 {
|
||||
sess.NewFlashErrorMessage(printer.Printf("alert.too_many_feeds_refresh"))
|
||||
} else {
|
||||
// We allow the end-user to force refresh all its feeds
|
||||
// without taking into consideration the number of errors.
|
||||
batchBuilder := h.store.NewBatchBuilder()
|
||||
batchBuilder.WithoutDisabledFeeds()
|
||||
batchBuilder.WithUserID(userID)
|
||||
|
||||
jobs, err := h.store.NewUserBatch(userID, h.store.CountFeeds(userID))
|
||||
jobs, err := batchBuilder.FetchJobs()
|
||||
if err != nil {
|
||||
html.ServerError(w, r, err)
|
||||
return
|
||||
|
|
@ -53,12 +60,9 @@ func (h *handler) refreshAllFeeds(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
go h.pool.Push(jobs)
|
||||
|
||||
sess := session.New(h.store, request.SessionID(r))
|
||||
view := view.New(h.tpl, r, sess)
|
||||
view.Set("menu", "feeds")
|
||||
view.Set("user", user)
|
||||
view.Set("countUnread", h.store.CountUnreadEntries(user.ID))
|
||||
view.Set("countErrorFeeds", h.store.CountUserFeedsWithErrors(user.ID))
|
||||
|
||||
html.OK(w, r, view.Render("feed_background_refresh"))
|
||||
sess.SetLastForceRefresh()
|
||||
sess.NewFlashMessage(printer.Printf("alert.background_feed_refresh"))
|
||||
}
|
||||
|
||||
html.Redirect(w, r, route.Path(h.router, "feeds"))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -119,6 +119,8 @@ func (m *middleware) handleAppSession(next http.Handler) http.Handler {
|
|||
ctx = context.WithValue(ctx, request.UserLanguageContextKey, session.Data.Language)
|
||||
ctx = context.WithValue(ctx, request.UserThemeContextKey, session.Data.Theme)
|
||||
ctx = context.WithValue(ctx, request.PocketRequestTokenContextKey, session.Data.PocketRequestToken)
|
||||
ctx = context.WithValue(ctx, request.LastForceRefreshContextKey, session.Data.LastForceRefresh)
|
||||
|
||||
next.ServeHTTP(w, r.WithContext(ctx))
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@
|
|||
package session // import "miniflux.app/v2/internal/ui/session"
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"miniflux.app/v2/internal/storage"
|
||||
)
|
||||
|
||||
|
|
@ -13,6 +15,15 @@ type Session struct {
|
|||
sessionID string
|
||||
}
|
||||
|
||||
// New returns a new session handler.
|
||||
func New(store *storage.Storage, sessionID string) *Session {
|
||||
return &Session{store, sessionID}
|
||||
}
|
||||
|
||||
func (s *Session) SetLastForceRefresh() {
|
||||
s.store.UpdateAppSessionField(s.sessionID, "last_force_refresh", time.Now().UTC().Unix())
|
||||
}
|
||||
|
||||
func (s *Session) SetOAuth2State(state string) {
|
||||
s.store.UpdateAppSessionField(s.sessionID, "oauth2_state", state)
|
||||
}
|
||||
|
|
@ -61,8 +72,3 @@ func (s *Session) SetTheme(theme string) {
|
|||
func (s *Session) SetPocketRequestToken(requestToken string) {
|
||||
s.store.UpdateAppSessionField(s.sessionID, "pocket_request_token", requestToken)
|
||||
}
|
||||
|
||||
// New returns a new session handler.
|
||||
func New(store *storage.Storage, sessionID string) *Session {
|
||||
return &Session{store, sessionID}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue