1
0
Fork 0

Add FILTER_ENTRY_MAX_AGE_DAYS config option to limit fetching all feed items

This commit is contained in:
Jean Khawand 2024-03-20 03:58:53 +01:00 committed by GitHub
parent 1ea3953271
commit a78d1c79da
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 53 additions and 4 deletions

View file

@ -1,7 +1,7 @@
version: '3.8'
services:
app:
image: mcr.microsoft.com/devcontainers/go
image: mcr.microsoft.com/devcontainers/go:1.22
volumes:
- ..:/workspace:cached
command: sleep infinity
@ -24,7 +24,7 @@ services:
ports:
- 5432:5432
apprise:
image: caronc/apprise:latest
image: caronc/apprise:1.0
restart: unless-stopped
hostname: apprise
volumes:

View file

@ -55,6 +55,7 @@ const (
defaultProxyOption = "http-only"
defaultProxyMediaTypes = "image"
defaultProxyUrl = ""
defaultFilterEntryMaxAgeDays = 0
defaultFetchOdyseeWatchTime = false
defaultFetchYouTubeWatchTime = false
defaultYouTubeEmbedUrlOverride = "https://www.youtube-nocookie.com/embed/"
@ -141,6 +142,7 @@ type Options struct {
proxyUrl string
fetchOdyseeWatchTime bool
fetchYouTubeWatchTime bool
filterEntryMaxAgeDays int
youTubeEmbedUrlOverride string
oauth2UserCreationAllowed bool
oauth2ClientID string
@ -213,6 +215,7 @@ func NewOptions() *Options {
proxyOption: defaultProxyOption,
proxyMediaTypes: []string{defaultProxyMediaTypes},
proxyUrl: defaultProxyUrl,
filterEntryMaxAgeDays: defaultFilterEntryMaxAgeDays,
fetchOdyseeWatchTime: defaultFetchOdyseeWatchTime,
fetchYouTubeWatchTime: defaultFetchYouTubeWatchTime,
youTubeEmbedUrlOverride: defaultYouTubeEmbedUrlOverride,
@ -612,6 +615,11 @@ func (o *Options) WebAuthn() bool {
return o.webAuthn
}
// FilterEntryMaxAgeDays returns the number of days after which entries should be retained.
func (o *Options) FilterEntryMaxAgeDays() int {
return o.filterEntryMaxAgeDays
}
// SortedOptions returns options as a list of key value pairs, sorted by keys.
func (o *Options) SortedOptions(redactSecret bool) []*Option {
var keyValues = map[string]interface{}{
@ -637,6 +645,7 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option {
"DISABLE_HSTS": !o.hsts,
"DISABLE_HTTP_SERVICE": !o.httpService,
"DISABLE_SCHEDULER_SERVICE": !o.schedulerService,
"FILTER_ENTRY_MAX_AGE_DAYS": o.filterEntryMaxAgeDays,
"FETCH_YOUTUBE_WATCH_TIME": o.fetchYouTubeWatchTime,
"FETCH_ODYSEE_WATCH_TIME": o.fetchOdyseeWatchTime,
"HTTPS": o.HTTPS,

View file

@ -112,6 +112,8 @@ func (p *Parser) parseLines(lines []string) (err error) {
p.opts.databaseMinConns = parseInt(value, defaultDatabaseMinConns)
case "DATABASE_CONNECTION_LIFETIME":
p.opts.databaseConnectionLifetime = parseInt(value, defaultDatabaseConnectionLifetime)
case "FILTER_ENTRY_MAX_AGE_DAYS":
p.opts.filterEntryMaxAgeDays = parseInt(value, defaultFilterEntryMaxAgeDays)
case "RUN_MIGRATIONS":
p.opts.runMigrations = parseBool(value, defaultRunMigrations)
case "DISABLE_HSTS":

View file

@ -47,8 +47,7 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us
slog.Int64("feed_id", feed.ID),
slog.String("feed_url", feed.FeedURL),
)
if isBlockedEntry(feed, entry) || !isAllowedEntry(feed, entry) {
if isBlockedEntry(feed, entry) || !isAllowedEntry(feed, entry) || !isRecentEntry(entry) {
continue
}
@ -413,3 +412,10 @@ func parseISO8601(from string) (time.Duration, error) {
return d, nil
}
func isRecentEntry(entry *model.Entry) bool {
if config.Opts.FilterEntryMaxAgeDays() == 0 || entry.Date.After(time.Now().AddDate(0, 0, -config.Opts.FilterEntryMaxAgeDays())) {
return true
}
return false
}

View file

@ -7,6 +7,7 @@ import (
"testing"
"time"
"miniflux.app/v2/internal/config"
"miniflux.app/v2/internal/model"
)
@ -92,3 +93,27 @@ func TestParseISO8601(t *testing.T) {
}
}
}
func TestIsRecentEntry(t *testing.T) {
parser := config.NewParser()
var err error
config.Opts, err = parser.ParseEnvironmentVariables()
if err != nil {
t.Fatalf(`Parsing failure: %v`, err)
}
var scenarios = []struct {
entry *model.Entry
expected bool
}{
{&model.Entry{Title: "Example1", Date: time.Date(2005, 5, 1, 05, 05, 05, 05, time.UTC)}, true},
{&model.Entry{Title: "Example2", Date: time.Date(2010, 5, 1, 05, 05, 05, 05, time.UTC)}, true},
{&model.Entry{Title: "Example3", Date: time.Date(2020, 5, 1, 05, 05, 05, 05, time.UTC)}, true},
{&model.Entry{Title: "Example4", Date: time.Date(2024, 3, 15, 05, 05, 05, 05, time.UTC)}, true},
}
for _, tc := range scenarios {
result := isRecentEntry(tc.entry)
if tc.expected != result {
t.Errorf(`Unexpected result, got %v for entry %q`, result, tc.entry.Title)
}
}
}

View file

@ -307,6 +307,13 @@ Set the value to 1 to disable the internal scheduler service\&.
.br
Default is false (The internal scheduler service is enabled)\&.
.TP
.B FILTER_ENTRY_MAX_AGE_DAYS
Number of days after which new entries should be retained.\&.
.br
Set 7 to fetch only entries 7 days old.\&.
.br
Default is 0\&.
.TP
.B CERT_FILE
Path to SSL certificate\&.
.br