diff --git a/model/feed.go b/model/feed.go index 020351e9..1994a7bd 100644 --- a/model/feed.go +++ b/model/feed.go @@ -15,8 +15,11 @@ import ( // List of supported schedulers. const ( - SchedulerRoundRobin = "round_robin" - SchedulerEntryFrequency = "entry_frequency" + SchedulerRoundRobin = "round_robin" + SchedulerEntryFrequency = "entry_frequency" + // Default settings for the feed query builder + DefaultFeedSorting = "parsing_error_count" + DefaultFeedSortingDirection = "desc" ) // Feed represents a feed in the application. diff --git a/storage/entry.go b/storage/entry.go index 56a0c069..deedb728 100644 --- a/storage/entry.go +++ b/storage/entry.go @@ -26,9 +26,9 @@ func (s *Storage) CountAllEntries() map[string]int64 { defer rows.Close() results := make(map[string]int64) - results["unread"] = 0 - results["read"] = 0 - results["removed"] = 0 + results[model.EntryStatusUnread] = 0 + results[model.EntryStatusRead] = 0 + results[model.EntryStatusRemoved] = 0 for rows.Next() { var status string @@ -41,7 +41,7 @@ func (s *Storage) CountAllEntries() map[string]int64 { results[status] = count } - results["total"] = results["unread"] + results["read"] + results["removed"] + results["total"] = results[model.EntryStatusUnread] + results[model.EntryStatusRead] + results[model.EntryStatusRemoved] return results } diff --git a/storage/entry_query_builder.go b/storage/entry_query_builder.go index c1204cf9..cbe1dad2 100644 --- a/storage/entry_query_builder.go +++ b/storage/entry_query_builder.go @@ -5,6 +5,7 @@ package storage // import "miniflux.app/storage" import ( + "database/sql" "fmt" "strings" "time" @@ -269,7 +270,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) { entries := make(model.Entries, 0) for rows.Next() { var entry model.Entry - var iconID interface{} + var iconID sql.NullInt64 var tz string entry.Feed = &model.Feed{} @@ -310,10 +311,10 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) { return nil, fmt.Errorf("unable to fetch entry row: %v", err) } - if iconID == nil { - entry.Feed.Icon.IconID = 0 + if iconID.Valid { + entry.Feed.Icon.IconID = iconID.Int64 } else { - entry.Feed.Icon.IconID = iconID.(int64) + entry.Feed.Icon.IconID = 0 } // Make sure that timestamp fields contains timezone information (API) diff --git a/storage/feed.go b/storage/feed.go index bf601427..c4524384 100644 --- a/storage/feed.go +++ b/storage/feed.go @@ -10,50 +10,8 @@ import ( "fmt" "miniflux.app/model" - "miniflux.app/timezone" ) -var feedListQuery = ` - SELECT - f.id, - f.feed_url, - f.site_url, - f.title, - f.etag_header, - f.last_modified_header, - f.user_id, - f.checked_at at time zone u.timezone, - f.parsing_error_count, - f.parsing_error_msg, - f.scraper_rules, - f.rewrite_rules, - f.blocklist_rules, - f.keeplist_rules, - f.crawler, - f.user_agent, - f.username, - f.password, - f.ignore_http_cache, - f.fetch_via_proxy, - f.disabled, - f.category_id, - c.title as category_title, - fi.icon_id, - u.timezone - FROM - feeds f - LEFT JOIN - categories c ON c.id=f.category_id - LEFT JOIN - feed_icons fi ON fi.feed_id=f.id - LEFT JOIN - users u ON u.id=f.user_id - WHERE - f.user_id=$1 - ORDER BY - f.parsing_error_count DESC, lower(f.title) ASC -` - // FeedExists checks if the given feed exists. func (s *Storage) FeedExists(userID, feedID int64) bool { var result bool @@ -146,193 +104,29 @@ func (s *Storage) CountAllFeedsWithErrors() int { // Feeds returns all feeds that belongs to the given user. func (s *Storage) Feeds(userID int64) (model.Feeds, error) { - return s.fetchFeeds(feedListQuery, "", userID) + builder := NewFeedQueryBuilder(s, userID) + builder.WithOrder(model.DefaultFeedSorting) + builder.WithDirection(model.DefaultFeedSortingDirection) + return builder.GetFeeds() } // FeedsWithCounters returns all feeds of the given user with counters of read and unread entries. func (s *Storage) FeedsWithCounters(userID int64) (model.Feeds, error) { - counterQuery := ` - SELECT - feed_id, - status, - count(*) - FROM - entries - WHERE - user_id=$1 AND status IN ('read', 'unread') - GROUP BY - feed_id, status - ` - return s.fetchFeeds(feedListQuery, counterQuery, userID) + builder := NewFeedQueryBuilder(s, userID) + builder.WithCounters() + builder.WithOrder(model.DefaultFeedSorting) + builder.WithDirection(model.DefaultFeedSortingDirection) + return builder.GetFeeds() } // FeedsByCategoryWithCounters returns all feeds of the given user/category with counters of read and unread entries. func (s *Storage) FeedsByCategoryWithCounters(userID, categoryID int64) (model.Feeds, error) { - feedQuery := ` - SELECT - f.id, - f.feed_url, - f.site_url, - f.title, - f.etag_header, - f.last_modified_header, - f.user_id, - f.checked_at at time zone u.timezone, - f.parsing_error_count, - f.parsing_error_msg, - f.scraper_rules, - f.rewrite_rules, - f.blocklist_rules, - f.keeplist_rules, - f.crawler, - f.user_agent, - f.username, - f.password, - f.ignore_http_cache, - f.fetch_via_proxy, - f.disabled, - f.category_id, - c.title as category_title, - fi.icon_id, - u.timezone - FROM - feeds f - LEFT JOIN - categories c ON c.id=f.category_id - LEFT JOIN - feed_icons fi ON fi.feed_id=f.id - LEFT JOIN - users u ON u.id=f.user_id - WHERE - f.user_id=$1 AND f.category_id=$2 - ORDER BY - f.parsing_error_count DESC, lower(f.title) ASC - ` - - counterQuery := ` - SELECT - e.feed_id, - e.status, - count(*) - FROM - entries e - LEFT JOIN - feeds f ON f.id=e.feed_id - WHERE - e.user_id=$1 AND f.category_id=$2 AND e.status IN ('read', 'unread') - GROUP BY - e.feed_id, e.status - ` - - return s.fetchFeeds(feedQuery, counterQuery, userID, categoryID) -} - -func (s *Storage) fetchFeedCounter(query string, args ...interface{}) (unreadCounters map[int64]int, readCounters map[int64]int, err error) { - rows, err := s.db.Query(query, args...) - if err != nil { - return nil, nil, fmt.Errorf(`store: unable to fetch feed counts: %v`, err) - } - defer rows.Close() - - readCounters = make(map[int64]int) - unreadCounters = make(map[int64]int) - for rows.Next() { - var feedID int64 - var status string - var count int - if err := rows.Scan(&feedID, &status, &count); err != nil { - return nil, nil, fmt.Errorf(`store: unable to fetch feed counter row: %v`, err) - } - - if status == "read" { - readCounters[feedID] = count - } else if status == "unread" { - unreadCounters[feedID] = count - } - } - - return readCounters, unreadCounters, nil -} - -func (s *Storage) fetchFeeds(feedQuery, counterQuery string, args ...interface{}) (model.Feeds, error) { - var ( - readCounters map[int64]int - unreadCounters map[int64]int - ) - - if counterQuery != "" { - var err error - readCounters, unreadCounters, err = s.fetchFeedCounter(counterQuery, args...) - if err != nil { - return nil, err - } - } - - feeds := make(model.Feeds, 0) - rows, err := s.db.Query(feedQuery, args...) - if err != nil { - return nil, fmt.Errorf(`store: unable to fetch feeds: %v`, err) - } - defer rows.Close() - - for rows.Next() { - var feed model.Feed - var iconID interface{} - var tz string - feed.Category = &model.Category{} - - err := rows.Scan( - &feed.ID, - &feed.FeedURL, - &feed.SiteURL, - &feed.Title, - &feed.EtagHeader, - &feed.LastModifiedHeader, - &feed.UserID, - &feed.CheckedAt, - &feed.ParsingErrorCount, - &feed.ParsingErrorMsg, - &feed.ScraperRules, - &feed.RewriteRules, - &feed.BlocklistRules, - &feed.KeeplistRules, - &feed.Crawler, - &feed.UserAgent, - &feed.Username, - &feed.Password, - &feed.IgnoreHTTPCache, - &feed.FetchViaProxy, - &feed.Disabled, - &feed.Category.ID, - &feed.Category.Title, - &iconID, - &tz, - ) - - if err != nil { - return nil, fmt.Errorf(`store: unable to fetch feeds row: %v`, err) - } - - if iconID != nil { - feed.Icon = &model.FeedIcon{FeedID: feed.ID, IconID: iconID.(int64)} - } - - if counterQuery != "" { - if count, found := readCounters[feed.ID]; found { - feed.ReadCount = count - } - - if count, found := unreadCounters[feed.ID]; found { - feed.UnreadCount = count - } - } - - feed.CheckedAt = timezone.Convert(tz, feed.CheckedAt) - feed.Category.UserID = feed.UserID - feeds = append(feeds, &feed) - } - - return feeds, nil + builder := NewFeedQueryBuilder(s, userID) + builder.WithCategoryID(categoryID) + builder.WithCounters() + builder.WithOrder(model.DefaultFeedSorting) + builder.WithDirection(model.DefaultFeedSortingDirection) + return builder.GetFeeds() } // WeeklyFeedEntryCount returns the weekly entry count for a feed. @@ -352,7 +146,7 @@ func (s *Storage) WeeklyFeedEntryCount(userID, feedID int64) (int, error) { err := s.db.QueryRow(query, userID, feedID).Scan(&weeklyCount) switch { - case err == sql.ErrNoRows: + case errors.Is(err, sql.ErrNoRows): return 0, nil case err != nil: return 0, fmt.Errorf(`store: unable to fetch weekly count for feed #%d: %v`, feedID, err) @@ -363,86 +157,18 @@ func (s *Storage) WeeklyFeedEntryCount(userID, feedID int64) (int, error) { // FeedByID returns a feed by the ID. func (s *Storage) FeedByID(userID, feedID int64) (*model.Feed, error) { - var feed model.Feed - var iconID interface{} - var tz string - feed.Category = &model.Category{UserID: userID} - - query := ` - SELECT - f.id, - f.feed_url, - f.site_url, - f.title, - f.etag_header, - f.last_modified_header, - f.user_id, f.checked_at at time zone u.timezone, - f.parsing_error_count, - f.parsing_error_msg, - f.scraper_rules, - f.rewrite_rules, - f.blocklist_rules, - f.keeplist_rules, - f.crawler, - f.user_agent, - f.username, - f.password, - f.ignore_http_cache, - f.fetch_via_proxy, - f.disabled, - f.category_id, - c.title as category_title, - fi.icon_id, - u.timezone - FROM feeds f - LEFT JOIN categories c ON c.id=f.category_id - LEFT JOIN feed_icons fi ON fi.feed_id=f.id - LEFT JOIN users u ON u.id=f.user_id - WHERE - f.user_id=$1 AND f.id=$2 - ` - - err := s.db.QueryRow(query, userID, feedID).Scan( - &feed.ID, - &feed.FeedURL, - &feed.SiteURL, - &feed.Title, - &feed.EtagHeader, - &feed.LastModifiedHeader, - &feed.UserID, - &feed.CheckedAt, - &feed.ParsingErrorCount, - &feed.ParsingErrorMsg, - &feed.ScraperRules, - &feed.RewriteRules, - &feed.BlocklistRules, - &feed.KeeplistRules, - &feed.Crawler, - &feed.UserAgent, - &feed.Username, - &feed.Password, - &feed.IgnoreHTTPCache, - &feed.FetchViaProxy, - &feed.Disabled, - &feed.Category.ID, - &feed.Category.Title, - &iconID, - &tz, - ) + builder := NewFeedQueryBuilder(s, userID) + builder.WithFeedID(feedID) + feed, err := builder.GetFeed() switch { - case err == sql.ErrNoRows: + case errors.Is(err, sql.ErrNoRows): return nil, nil case err != nil: return nil, fmt.Errorf(`store: unable to fetch feed #%d: %v`, feedID, err) } - - if iconID != nil { - feed.Icon = &model.FeedIcon{FeedID: feed.ID, IconID: iconID.(int64)} - } - - feed.CheckedAt = timezone.Convert(tz, feed.CheckedAt) - return &feed, nil + + return feed, nil } // CreateFeed creates a new feed. diff --git a/storage/feed_query_builder.go b/storage/feed_query_builder.go new file mode 100644 index 00000000..5b62fb13 --- /dev/null +++ b/storage/feed_query_builder.go @@ -0,0 +1,307 @@ +// Copyright 2021 Frédéric Guillot. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +package storage // import "miniflux.app/storage" + +import ( + "database/sql" + "fmt" + "strings" + + "miniflux.app/model" + "miniflux.app/timezone" +) + +// FeedQueryBuilder builds a SQL query to fetch feeds. +type FeedQueryBuilder struct { + store *Storage + args []interface{} + conditions []string + order string + direction string + limit int + offset int + withCounters bool + counterJoinFeeds bool + counterArgs []interface{} + counterConditions []string +} + +// NewFeedQueryBuilder returns a new FeedQueryBuilder. +func NewFeedQueryBuilder(store *Storage, userID int64) *FeedQueryBuilder { + return &FeedQueryBuilder{ + store: store, + args: []interface{}{userID}, + conditions: []string{"f.user_id = $1"}, + counterArgs: []interface{}{userID, model.EntryStatusRead, model.EntryStatusUnread}, + counterConditions: []string{"e.user_id = $1", "e.status IN ($2, $3)"}, + } +} + +// WithCategoryID filter by category ID. +func (f *FeedQueryBuilder) WithCategoryID(categoryID int64) *FeedQueryBuilder { + if categoryID > 0 { + f.conditions = append(f.conditions, fmt.Sprintf("f.category_id = $%d", len(f.args)+1)) + f.args = append(f.args, categoryID) + f.counterConditions = append(f.counterConditions, fmt.Sprintf("f.category_id = $%d", len(f.counterArgs)+1)) + f.counterArgs = append(f.counterArgs, categoryID) + f.counterJoinFeeds = true + } + return f +} + +// WithFeedID filter by feed ID. +func (f *FeedQueryBuilder) WithFeedID(feedID int64) *FeedQueryBuilder { + if feedID > 0 { + f.conditions = append(f.conditions, fmt.Sprintf("f.id = $%d", len(f.args)+1)) + f.args = append(f.args, feedID) + } + return f +} + +// WithCounters let the builder return feeds with counters of statuses of entries. +func (f *FeedQueryBuilder) WithCounters() *FeedQueryBuilder { + f.withCounters = true + return f +} + +// WithOrder set the sorting order. +func (f *FeedQueryBuilder) WithOrder(order string) *FeedQueryBuilder { + f.order = order + return f +} + +// WithDirection set the sorting direction. +func (f *FeedQueryBuilder) WithDirection(direction string) *FeedQueryBuilder { + f.direction = direction + return f +} + +// WithLimit set the limit. +func (f *FeedQueryBuilder) WithLimit(limit int) *FeedQueryBuilder { + f.limit = limit + return f +} + +// WithOffset set the offset. +func (f *FeedQueryBuilder) WithOffset(offset int) *FeedQueryBuilder { + f.offset = offset + return f +} + +func (f *FeedQueryBuilder) buildCondition() string { + return strings.Join(f.conditions, " AND ") +} + +func (f *FeedQueryBuilder) buildCounterCondition() string { + return strings.Join(f.counterConditions, " AND ") +} + +func (f *FeedQueryBuilder) buildSorting() string { + var parts []string + + if f.order != "" { + parts = append(parts, fmt.Sprintf(`ORDER BY %s`, f.order)) + } + + if f.direction != "" { + parts = append(parts, fmt.Sprintf(`%s`, f.direction)) + } + + if len(parts) > 0 { + parts = append(parts, ", lower(f.title) ASC") + } + + if f.limit > 0 { + parts = append(parts, fmt.Sprintf(`LIMIT %d`, f.limit)) + } + + if f.offset > 0 { + parts = append(parts, fmt.Sprintf(`OFFSET %d`, f.offset)) + } + + return strings.Join(parts, " ") +} + +// GetFeed returns a single feed that match the condition. +func (f *FeedQueryBuilder) GetFeed() (*model.Feed, error) { + f.limit = 1 + feeds, err := f.GetFeeds() + if err != nil { + return nil, err + } + + if len(feeds) != 1 { + return nil, nil + } + + return feeds[0], nil +} + +// GetFeeds returns a list of feeds that match the condition. +func (f *FeedQueryBuilder) GetFeeds() (model.Feeds, error) { + var query = ` + SELECT + f.id, + f.feed_url, + f.site_url, + f.title, + f.etag_header, + f.last_modified_header, + f.user_id, + f.checked_at at time zone u.timezone, + f.parsing_error_count, + f.parsing_error_msg, + f.scraper_rules, + f.rewrite_rules, + f.blocklist_rules, + f.keeplist_rules, + f.crawler, + f.user_agent, + f.username, + f.password, + f.ignore_http_cache, + f.fetch_via_proxy, + f.disabled, + f.category_id, + c.title as category_title, + fi.icon_id, + u.timezone + FROM + feeds f + LEFT JOIN + categories c ON c.id=f.category_id + LEFT JOIN + feed_icons fi ON fi.feed_id=f.id + LEFT JOIN + users u ON u.id=f.user_id + WHERE %s + %s + ` + + query = fmt.Sprintf(query, f.buildCondition(), f.buildSorting()) + + rows, err := f.store.db.Query(query, f.args...) + if err != nil { + return nil, fmt.Errorf(`store: unable to fetch feeds: %w`, err) + } + defer rows.Close() + + readCounters, unreadCounters, err := f.fetchFeedCounter() + if err != nil { + return nil, err + } + + feeds := make(model.Feeds, 0) + for rows.Next() { + var feed model.Feed + var iconID sql.NullInt64 + var tz string + feed.Category = &model.Category{} + + err := rows.Scan( + &feed.ID, + &feed.FeedURL, + &feed.SiteURL, + &feed.Title, + &feed.EtagHeader, + &feed.LastModifiedHeader, + &feed.UserID, + &feed.CheckedAt, + &feed.ParsingErrorCount, + &feed.ParsingErrorMsg, + &feed.ScraperRules, + &feed.RewriteRules, + &feed.BlocklistRules, + &feed.KeeplistRules, + &feed.Crawler, + &feed.UserAgent, + &feed.Username, + &feed.Password, + &feed.IgnoreHTTPCache, + &feed.FetchViaProxy, + &feed.Disabled, + &feed.Category.ID, + &feed.Category.Title, + &iconID, + &tz, + ) + + if err != nil { + return nil, fmt.Errorf(`store: unable to fetch feeds row: %w`, err) + } + + if iconID.Valid { + feed.Icon = &model.FeedIcon{FeedID: feed.ID, IconID: iconID.Int64} + } else { + feed.Icon = &model.FeedIcon{FeedID: feed.ID, IconID: 0} + } + + if readCounters != nil { + if count, found := readCounters[feed.ID]; found { + feed.ReadCount = count + } + } + if unreadCounters != nil { + if count, found := unreadCounters[feed.ID]; found { + feed.UnreadCount = count + } + } + + feed.CheckedAt = timezone.Convert(tz, feed.CheckedAt) + feed.Category.UserID = feed.UserID + feeds = append(feeds, &feed) + } + + return feeds, nil +} + +func (f *FeedQueryBuilder) fetchFeedCounter() (unreadCounters map[int64]int, readCounters map[int64]int, err error) { + if !f.withCounters { + return nil, nil, nil + } + query := ` + SELECT + e.feed_id, + e.status, + count(*) + FROM + entries e + %s + WHERE + %s + GROUP BY + e.feed_id, e.status + ` + join := "" + if f.counterJoinFeeds { + join = "LEFT JOIN feeds f ON f.id=e.feed_id" + } + query = fmt.Sprintf(query, join, f.buildCounterCondition()) + + rows, err := f.store.db.Query(query, f.counterArgs...) + if err != nil { + return nil, nil, fmt.Errorf(`store: unable to fetch feed counts: %w`, err) + } + defer rows.Close() + + readCounters = make(map[int64]int) + unreadCounters = make(map[int64]int) + for rows.Next() { + var feedID int64 + var status string + var count int + if err := rows.Scan(&feedID, &status, &count); err != nil { + return nil, nil, fmt.Errorf(`store: unable to fetch feed counter row: %w`, err) + } + + if status == model.EntryStatusRead { + readCounters[feedID] = count + } else if status == model.EntryStatusUnread { + unreadCounters[feedID] = count + } + } + + return readCounters, unreadCounters, nil +} \ No newline at end of file