1
0
Fork 0
miniflux/internal/storage/entry_query_builder.go

463 lines
12 KiB
Go
Raw Normal View History

// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
2017-11-20 00:10:04 -05:00
package storage // import "miniflux.app/v2/internal/storage"
2017-11-20 00:10:04 -05:00
import (
2021-01-18 16:22:09 -05:00
"database/sql"
2017-11-20 00:10:04 -05:00
"fmt"
"strings"
"time"
2017-12-03 20:44:27 -05:00
"github.com/lib/pq"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/timezone"
2017-11-20 00:10:04 -05:00
)
// EntryQueryBuilder builds a SQL query to fetch entries.
2017-11-20 00:10:04 -05:00
type EntryQueryBuilder struct {
store *Storage
args []interface{}
conditions []string
sortExpressions []string
limit int
offset int
fetchEnclosures bool
}
// WithEnclosures fetches enclosures for each entry.
func (e *EntryQueryBuilder) WithEnclosures() *EntryQueryBuilder {
e.fetchEnclosures = true
return e
2017-12-22 14:33:01 -05:00
}
// WithSearchQuery adds full-text search query to the condition.
func (e *EntryQueryBuilder) WithSearchQuery(query string) *EntryQueryBuilder {
if query != "" {
nArgs := len(e.args) + 1
e.conditions = append(e.conditions, fmt.Sprintf("e.document_vectors @@ plainto_tsquery($%d)", nArgs))
e.args = append(e.args, query)
// 0.0000001 = 0.1 / (seconds_in_a_day)
e.WithSorting(
fmt.Sprintf("ts_rank(document_vectors, plainto_tsquery($%d)) - extract (epoch from now() - published_at)::float * 0.0000001", nArgs),
"DESC",
)
}
return e
}
2017-12-22 14:33:01 -05:00
// WithStarred adds starred filter.
func (e *EntryQueryBuilder) WithStarred(starred bool) *EntryQueryBuilder {
if starred {
e.conditions = append(e.conditions, "e.starred is true")
} else {
e.conditions = append(e.conditions, "e.starred is false")
}
2017-12-22 14:33:01 -05:00
return e
2017-12-03 20:44:27 -05:00
}
// BeforeChangedDate adds a condition < changed_at
func (e *EntryQueryBuilder) BeforeChangedDate(date time.Time) *EntryQueryBuilder {
e.conditions = append(e.conditions, fmt.Sprintf("e.changed_at < $%d", len(e.args)+1))
e.args = append(e.args, date)
return e
}
// AfterChangedDate adds a condition > changed_at
func (e *EntryQueryBuilder) AfterChangedDate(date time.Time) *EntryQueryBuilder {
e.conditions = append(e.conditions, fmt.Sprintf("e.changed_at > $%d", len(e.args)+1))
e.args = append(e.args, date)
return e
}
// BeforePublishedDate adds a condition < published_at
func (e *EntryQueryBuilder) BeforePublishedDate(date time.Time) *EntryQueryBuilder {
2018-06-08 21:24:41 -04:00
e.conditions = append(e.conditions, fmt.Sprintf("e.published_at < $%d", len(e.args)+1))
e.args = append(e.args, date)
2017-12-03 20:44:27 -05:00
return e
}
// AfterPublishedDate adds a condition > published_at
func (e *EntryQueryBuilder) AfterPublishedDate(date time.Time) *EntryQueryBuilder {
e.conditions = append(e.conditions, fmt.Sprintf("e.published_at > $%d", len(e.args)+1))
e.args = append(e.args, date)
return e
}
// BeforeEntryID adds a condition < entryID.
func (e *EntryQueryBuilder) BeforeEntryID(entryID int64) *EntryQueryBuilder {
if entryID != 0 {
e.conditions = append(e.conditions, fmt.Sprintf("e.id < $%d", len(e.args)+1))
e.args = append(e.args, entryID)
}
return e
}
// AfterEntryID adds a condition > entryID.
func (e *EntryQueryBuilder) AfterEntryID(entryID int64) *EntryQueryBuilder {
2018-06-08 21:24:41 -04:00
if entryID != 0 {
e.conditions = append(e.conditions, fmt.Sprintf("e.id > $%d", len(e.args)+1))
e.args = append(e.args, entryID)
}
2017-12-03 20:44:27 -05:00
return e
}
// WithEntryIDs filter by entry IDs.
2017-12-03 20:44:27 -05:00
func (e *EntryQueryBuilder) WithEntryIDs(entryIDs []int64) *EntryQueryBuilder {
2018-06-08 21:24:41 -04:00
e.conditions = append(e.conditions, fmt.Sprintf("e.id = ANY($%d)", len(e.args)+1))
e.args = append(e.args, pq.Int64Array(entryIDs))
2017-12-03 20:44:27 -05:00
return e
2017-11-20 00:10:04 -05:00
}
// WithEntryID filter by entry ID.
2017-11-20 00:10:04 -05:00
func (e *EntryQueryBuilder) WithEntryID(entryID int64) *EntryQueryBuilder {
2018-06-08 21:24:41 -04:00
if entryID != 0 {
e.conditions = append(e.conditions, fmt.Sprintf("e.id = $%d", len(e.args)+1))
e.args = append(e.args, entryID)
}
2017-11-20 00:10:04 -05:00
return e
}
// WithFeedID filter by feed ID.
2017-11-20 00:10:04 -05:00
func (e *EntryQueryBuilder) WithFeedID(feedID int64) *EntryQueryBuilder {
if feedID > 0 {
2018-06-08 21:24:41 -04:00
e.conditions = append(e.conditions, fmt.Sprintf("e.feed_id = $%d", len(e.args)+1))
e.args = append(e.args, feedID)
}
2017-11-20 00:10:04 -05:00
return e
}
// WithCategoryID filter by category ID.
2017-11-20 00:10:04 -05:00
func (e *EntryQueryBuilder) WithCategoryID(categoryID int64) *EntryQueryBuilder {
if categoryID > 0 {
2018-06-08 21:24:41 -04:00
e.conditions = append(e.conditions, fmt.Sprintf("f.category_id = $%d", len(e.args)+1))
e.args = append(e.args, categoryID)
}
2017-11-20 00:10:04 -05:00
return e
}
// WithStatus filter by entry status.
2017-11-20 00:10:04 -05:00
func (e *EntryQueryBuilder) WithStatus(status string) *EntryQueryBuilder {
2018-06-08 21:24:41 -04:00
if status != "" {
e.conditions = append(e.conditions, fmt.Sprintf("e.status = $%d", len(e.args)+1))
e.args = append(e.args, status)
}
2017-11-20 00:10:04 -05:00
return e
}
// WithStatuses filter by a list of entry statuses.
func (e *EntryQueryBuilder) WithStatuses(statuses []string) *EntryQueryBuilder {
if len(statuses) > 0 {
e.conditions = append(e.conditions, fmt.Sprintf("e.status = ANY($%d)", len(e.args)+1))
e.args = append(e.args, pq.StringArray(statuses))
}
return e
}
// WithTags filter by a list of entry tags.
func (e *EntryQueryBuilder) WithTags(tags []string) *EntryQueryBuilder {
if len(tags) > 0 {
for _, cat := range tags {
e.conditions = append(e.conditions, fmt.Sprintf("$%d = ANY(e.tags)", len(e.args)+1))
e.args = append(e.args, cat)
}
}
return e
}
// WithoutStatus set the entry status that should not be returned.
func (e *EntryQueryBuilder) WithoutStatus(status string) *EntryQueryBuilder {
2018-06-08 21:24:41 -04:00
if status != "" {
e.conditions = append(e.conditions, fmt.Sprintf("e.status <> $%d", len(e.args)+1))
e.args = append(e.args, status)
}
return e
}
// WithShareCode set the entry share code.
func (e *EntryQueryBuilder) WithShareCode(shareCode string) *EntryQueryBuilder {
e.conditions = append(e.conditions, fmt.Sprintf("e.share_code = $%d", len(e.args)+1))
e.args = append(e.args, shareCode)
return e
}
// WithShareCodeNotEmpty adds a filter for non-empty share code.
func (e *EntryQueryBuilder) WithShareCodeNotEmpty() *EntryQueryBuilder {
e.conditions = append(e.conditions, "e.share_code <> ''")
return e
}
// WithSorting add a sort expression.
func (e *EntryQueryBuilder) WithSorting(column, direction string) *EntryQueryBuilder {
e.sortExpressions = append(e.sortExpressions, fmt.Sprintf("%s %s", column, direction))
2017-11-20 00:10:04 -05:00
return e
}
// WithLimit set the limit.
2017-11-20 00:10:04 -05:00
func (e *EntryQueryBuilder) WithLimit(limit int) *EntryQueryBuilder {
if limit > 0 {
e.limit = limit
}
2017-11-20 00:10:04 -05:00
return e
}
// WithOffset set the offset.
2017-11-20 00:10:04 -05:00
func (e *EntryQueryBuilder) WithOffset(offset int) *EntryQueryBuilder {
if offset > 0 {
e.offset = offset
}
2017-11-20 00:10:04 -05:00
return e
}
func (e *EntryQueryBuilder) WithGloballyVisible() *EntryQueryBuilder {
2023-10-17 22:45:08 -04:00
e.conditions = append(e.conditions, "c.hide_globally IS FALSE")
e.conditions = append(e.conditions, "f.hide_globally IS FALSE")
return e
}
// CountEntries count the number of entries that match the condition.
2017-11-20 00:10:04 -05:00
func (e *EntryQueryBuilder) CountEntries() (count int, err error) {
query := `
SELECT count(*)
FROM entries e
JOIN feeds f ON f.id = e.feed_id
JOIN categories c ON c.id = f.category_id
WHERE %s
`
2018-06-08 21:24:41 -04:00
condition := e.buildCondition()
err = e.store.db.QueryRow(fmt.Sprintf(query, condition), e.args...).Scan(&count)
2017-11-20 00:10:04 -05:00
if err != nil {
return 0, fmt.Errorf("store: unable to count entries: %v", err)
2017-11-20 00:10:04 -05:00
}
return count, nil
}
// GetEntry returns a single entry that match the condition.
2017-11-20 00:10:04 -05:00
func (e *EntryQueryBuilder) GetEntry() (*model.Entry, error) {
e.limit = 1
entries, err := e.GetEntries()
if err != nil {
return nil, err
}
if len(entries) != 1 {
return nil, nil
}
entries[0].Enclosures, err = e.store.GetEnclosures(entries[0].ID)
if err != nil {
return nil, err
}
return entries[0], nil
}
// GetEntries returns a list of entries that match the condition.
2017-11-20 00:10:04 -05:00
func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) {
query := `
SELECT
e.id,
e.user_id,
e.feed_id,
e.hash,
e.published_at at time zone u.timezone,
e.title,
e.url,
e.comments_url,
e.author,
e.share_code,
e.content,
e.status,
e.starred,
e.reading_time,
2020-11-29 20:04:18 -05:00
e.created_at,
e.changed_at,
e.tags,
(SELECT true FROM enclosures WHERE entry_id=e.id LIMIT 1) as has_enclosure,
f.title as feed_title,
f.feed_url,
f.site_url,
f.checked_at,
f.category_id,
c.title as category_title,
c.hide_globally as category_hidden,
f.scraper_rules,
f.rewrite_rules,
f.crawler,
f.user_agent,
2021-03-22 23:27:58 -04:00
f.cookie,
f.hide_globally,
Add Media Player and resume to last playback position In order to ease podcast listening, the player can be put on top of the feed entry as main content. Use the `Use podcast player` option to enable that. It works on audio and video. Also, when playing audio or video, progression will be saved in order to be able to resume listening later. This position saving is done using the original attachement/enclosures player AND podcast player and do not rely on the podcast player option ti be enabled. Additionally, I made the player fill the width with the entry container to ease seeking and have a bigger video. updateEnclosures now keep existing enclosures based on URL When feeds get updated, enclosures entries are always wiped and re-created. This cause two issue - enclosure progression get lost in the process - enclosure ID changes I used the URL as identifier of an enclosure. Not perfect but hopefully should work. When an enclosure already exist, I simply do nothing and leave the entry as is in the database. If anyone is listening/watching to this enclosure during the refresh, the id stay coherent and progression saving still works. The updateEnclosures function got a bit more complex. I tried to make it the more clear I could. Some optimisation are possible but would make the function harder to read in my opinion. I'm not sure if this is often the case, but some feeds may include tracking or simply change the url each time we update the feed. In those situation, enclosures ids and progression will be lost. I have no idea how to handle this last situation. Use the size instead/alongside url to define the identity of an enclosure ? Translation: english as placeholder for every language except French Aside, I tested a video feed and fixed a few things for it. In fact, the MimeType was not working at all on my side, and found a pretty old stackoverflow discussion that suggest to use an Apple non-standard MimeType for m4v video format. I only did one substitution because I only have one feed to test. Any new video feed can make this go away or evolve depending on the situation. Real video feeds does not tend to be easy to find and test extensively this. Co-authored-by: toastal
2023-04-13 05:46:43 -04:00
f.no_media_player,
fi.icon_id,
u.timezone
FROM
entries e
LEFT JOIN
feeds f ON f.id=e.feed_id
LEFT JOIN
categories c ON c.id=f.category_id
LEFT JOIN
feed_icons fi ON fi.feed_id=f.id
LEFT JOIN
users u ON u.id=e.user_id
2017-11-20 00:10:04 -05:00
WHERE %s %s
`
2018-06-08 21:24:41 -04:00
condition := e.buildCondition()
sorting := e.buildSorting()
query = fmt.Sprintf(query, condition, sorting)
2017-11-20 00:10:04 -05:00
2018-06-08 21:24:41 -04:00
rows, err := e.store.db.Query(query, e.args...)
2017-11-20 00:10:04 -05:00
if err != nil {
return nil, fmt.Errorf("store: unable to get entries: %v", err)
2017-11-20 00:10:04 -05:00
}
defer rows.Close()
entries := make(model.Entries, 0)
for rows.Next() {
2021-01-18 16:22:09 -05:00
var iconID sql.NullInt64
var tz string
var hasEnclosure sql.NullBool
2017-11-20 00:10:04 -05:00
entry := model.NewEntry()
2017-11-20 00:10:04 -05:00
err := rows.Scan(
&entry.ID,
&entry.UserID,
&entry.FeedID,
&entry.Hash,
&entry.Date,
&entry.Title,
&entry.URL,
2018-04-07 16:50:45 -04:00
&entry.CommentsURL,
2017-11-20 00:10:04 -05:00
&entry.Author,
&entry.ShareCode,
2017-11-20 00:10:04 -05:00
&entry.Content,
&entry.Status,
2017-12-22 14:33:01 -05:00
&entry.Starred,
&entry.ReadingTime,
2020-11-29 20:04:18 -05:00
&entry.CreatedAt,
&entry.ChangedAt,
pq.Array(&entry.Tags),
&hasEnclosure,
2017-11-20 00:10:04 -05:00
&entry.Feed.Title,
&entry.Feed.FeedURL,
&entry.Feed.SiteURL,
&entry.Feed.CheckedAt,
&entry.Feed.Category.ID,
&entry.Feed.Category.Title,
&entry.Feed.Category.HideGlobally,
2017-12-10 23:51:04 -05:00
&entry.Feed.ScraperRules,
&entry.Feed.RewriteRules,
&entry.Feed.Crawler,
&entry.Feed.UserAgent,
2021-03-22 23:27:58 -04:00
&entry.Feed.Cookie,
&entry.Feed.HideGlobally,
Add Media Player and resume to last playback position In order to ease podcast listening, the player can be put on top of the feed entry as main content. Use the `Use podcast player` option to enable that. It works on audio and video. Also, when playing audio or video, progression will be saved in order to be able to resume listening later. This position saving is done using the original attachement/enclosures player AND podcast player and do not rely on the podcast player option ti be enabled. Additionally, I made the player fill the width with the entry container to ease seeking and have a bigger video. updateEnclosures now keep existing enclosures based on URL When feeds get updated, enclosures entries are always wiped and re-created. This cause two issue - enclosure progression get lost in the process - enclosure ID changes I used the URL as identifier of an enclosure. Not perfect but hopefully should work. When an enclosure already exist, I simply do nothing and leave the entry as is in the database. If anyone is listening/watching to this enclosure during the refresh, the id stay coherent and progression saving still works. The updateEnclosures function got a bit more complex. I tried to make it the more clear I could. Some optimisation are possible but would make the function harder to read in my opinion. I'm not sure if this is often the case, but some feeds may include tracking or simply change the url each time we update the feed. In those situation, enclosures ids and progression will be lost. I have no idea how to handle this last situation. Use the size instead/alongside url to define the identity of an enclosure ? Translation: english as placeholder for every language except French Aside, I tested a video feed and fixed a few things for it. In fact, the MimeType was not working at all on my side, and found a pretty old stackoverflow discussion that suggest to use an Apple non-standard MimeType for m4v video format. I only did one substitution because I only have one feed to test. Any new video feed can make this go away or evolve depending on the situation. Real video feeds does not tend to be easy to find and test extensively this. Co-authored-by: toastal
2023-04-13 05:46:43 -04:00
&entry.Feed.NoMediaPlayer,
2017-11-20 00:10:04 -05:00
&iconID,
&tz,
2017-11-20 00:10:04 -05:00
)
if err != nil {
return nil, fmt.Errorf("store: unable to fetch entry row: %v", err)
}
if hasEnclosure.Valid && hasEnclosure.Bool && e.fetchEnclosures {
entry.Enclosures, err = e.store.GetEnclosures(entry.ID)
if err != nil {
return nil, fmt.Errorf("store: unable to fetch enclosures for entry #%d: %w", entry.ID, err)
}
2017-11-20 00:10:04 -05:00
}
2021-01-18 16:22:09 -05:00
if iconID.Valid {
entry.Feed.Icon.IconID = iconID.Int64
2017-11-20 00:10:04 -05:00
} else {
2021-01-18 16:22:09 -05:00
entry.Feed.Icon.IconID = 0
2017-11-20 00:10:04 -05:00
}
// Make sure that timestamp fields contains timezone information (API)
entry.Date = timezone.Convert(tz, entry.Date)
2020-11-29 20:04:18 -05:00
entry.CreatedAt = timezone.Convert(tz, entry.CreatedAt)
entry.ChangedAt = timezone.Convert(tz, entry.ChangedAt)
entry.Feed.CheckedAt = timezone.Convert(tz, entry.Feed.CheckedAt)
2017-11-20 00:10:04 -05:00
entry.Feed.ID = entry.FeedID
2018-06-08 21:24:41 -04:00
entry.Feed.UserID = entry.UserID
2017-11-20 00:10:04 -05:00
entry.Feed.Icon.FeedID = entry.FeedID
2018-06-08 21:24:41 -04:00
entry.Feed.Category.UserID = entry.UserID
entries = append(entries, entry)
2017-11-20 00:10:04 -05:00
}
return entries, nil
}
2017-12-03 20:44:27 -05:00
// GetEntryIDs returns a list of entry IDs that match the condition.
func (e *EntryQueryBuilder) GetEntryIDs() ([]int64, error) {
2018-06-08 21:24:41 -04:00
query := `SELECT e.id FROM entries e LEFT JOIN feeds f ON f.id=e.feed_id WHERE %s %s`
2017-12-03 20:44:27 -05:00
2018-06-08 21:24:41 -04:00
condition := e.buildCondition()
query = fmt.Sprintf(query, condition, e.buildSorting())
rows, err := e.store.db.Query(query, e.args...)
2017-12-03 20:44:27 -05:00
if err != nil {
return nil, fmt.Errorf("store: unable to get entries: %v", err)
2017-12-03 20:44:27 -05:00
}
defer rows.Close()
var entryIDs []int64
for rows.Next() {
var entryID int64
err := rows.Scan(&entryID)
if err != nil {
return nil, fmt.Errorf("store: unable to fetch entry row: %v", err)
2017-12-03 20:44:27 -05:00
}
entryIDs = append(entryIDs, entryID)
}
return entryIDs, nil
}
2018-06-08 21:24:41 -04:00
func (e *EntryQueryBuilder) buildCondition() string {
return strings.Join(e.conditions, " AND ")
2017-11-20 00:10:04 -05:00
}
func (e *EntryQueryBuilder) buildSorting() string {
var parts []string
2017-11-20 00:10:04 -05:00
if len(e.sortExpressions) > 0 {
parts = append(parts, fmt.Sprintf(`ORDER BY %s`, strings.Join(e.sortExpressions, ", ")))
2017-11-20 00:10:04 -05:00
}
if e.limit > 0 {
parts = append(parts, fmt.Sprintf(`LIMIT %d`, e.limit))
2017-11-20 00:10:04 -05:00
}
if e.offset > 0 {
parts = append(parts, fmt.Sprintf(`OFFSET %d`, e.offset))
2017-11-20 00:10:04 -05:00
}
return strings.Join(parts, " ")
2017-11-20 00:10:04 -05:00
}
// NewEntryQueryBuilder returns a new EntryQueryBuilder.
2017-12-28 22:20:14 -05:00
func NewEntryQueryBuilder(store *Storage, userID int64) *EntryQueryBuilder {
2017-11-20 00:10:04 -05:00
return &EntryQueryBuilder{
2018-06-08 21:24:41 -04:00
store: store,
args: []interface{}{userID},
conditions: []string{"e.user_id = $1"},
2017-11-20 00:10:04 -05:00
}
}
// NewAnonymousQueryBuilder returns a new EntryQueryBuilder suitable for anonymous users.
func NewAnonymousQueryBuilder(store *Storage) *EntryQueryBuilder {
return &EntryQueryBuilder{
store: store,
}
}