forks/miniflux
1
0
Fork 0
Code Releases Activity

Add full-text search for entries and add search parameter to the API

Browse source
This commit is contained in:
Frédéric Guillot 2018-07-04 17:40:03 -07:00
parent 89e5dacca9
commit af15412954
7 changed files with 75 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
api/entry.go
View file

@ -248,4 +248,9 @@ func configureFilters(builder *storage.EntryQueryBuilder, r *http.Request) {
if request.HasQueryParam(r, "starred") {
builder.WithStarred()
}
searchQuery := request.QueryParam(r, "search", "")
if searchQuery != "" {
builder.WithSearchQuery(searchQuery)
}
}

8
locale/translations.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2018-06-30 23:41:46.05193077 +0200 CEST m=+0.307244685
// 2018-07-04 14:42:27.494264089 -0700 PDT m=+0.053526807
package locale
@ -580,7 +580,7 @@ var translations = map[string]string{
"Yes": "Ja",
"No": "Nee",
"This feed already exists (%s)": "Deze feed bestaat al (%s)",
"Unable to fetch feed (Status Code = %d)": "Kon feed niet update (code=%d)",
"Unable to fetch feed (Status Code = %d)": "Kon feed niet updaten (statuscode = %d)",
"Unable to open this link: %v": "Kon link niet volgen: %v",
"Unable to analyze this page: %v": "Kon pagina niet analyseren: %v",
"Unable to find any subscription.": "Kon geen feeds vinden.",
@ -650,7 +650,7 @@ var translations = map[string]string{
"Scraper Rules": "Scraper regels",
"Rewrite Rules": "Rewrite regels",
"Preferences saved!": "Instellingen opgeslagen!",
"Your external account is now linked!": "Jouw externe account is nu gekoppeld!",
"Your external account is now linked!": "Je externe account is nu gekoppeld!",
"Save articles to Wallabag": "Sauvegarder les articles vers Wallabag",
"Wallabag API Endpoint": "Wallabag URL",
"Wallabag Client ID": "Wallabag Client-ID",
@ -1167,7 +1167,7 @@ var translationsChecksums = map[string]string{
"de_DE": "eddbb2c3224169a6533eed6f917af95b8d9bee58c3b3d61951260face7edd768",
"en_US": "6fe95384260941e8a5a3c695a655a932e0a8a6a572c1e45cb2b1ae8baa01b897",
"fr_FR": "7a451a1d09e051a847f554937e07a6af24b92f1da7f46da8c0ef2d4cc31bcec6",
"nl_NL": "bec647fdfb325a30050c50bdb9a29fee58b0705109d32c09b5ace77aa0777e7c",
"nl_NL": "05cca4936bd3b0fa44057c4dab64acdef3aed32fbb682393f254cfe2f686ef1f",
"pl_PL": "2295f35a98c8f60cfc6bab241d26b224c06979cc9ca3740bb89c63c7596a0431",
"zh_CN": "f5fb0a9b7336c51e74d727a2fb294bab3514e3002376da7fd904e0d7caed1a1c",
}

3
sql/schema_version_20.sql Normal file
View file

@ -0,0 +1,3 @@
alter table entries add column document_vectors tsvector;
update entries set document_vectors = to_tsvector(title || ' ' || coalesce(content, ''));
create index document_vectors_idx on entries using gin(document_vectors);

6
sql/sql.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT.
// 2018-06-19 22:56:40.283528941 -0700 PDT m=+0.005341061
// 2018-07-04 14:42:27.443758746 -0700 PDT m=+0.003021464
package sql
@ -142,6 +142,9 @@ alter table feeds add column password text default '';`,
alter table users add column extra hstore;
create index users_extra_idx on users using gin(extra);
`,
"schema_version_20": `alter table entries add column document_vectors tsvector;
update entries set document_vectors = to_tsvector(title || ' ' || coalesce(content, ''));
create index document_vectors_idx on entries using gin(document_vectors);`,
"schema_version_3": `create table tokens (
id text not null,
value text not null,
@ -189,6 +192,7 @@ var SqlMapChecksums = map[string]string{
"schema_version_18": "c0ec24847612c7f2dc326cf735baffba79391a56aedd73292371a39f38724a71",
"schema_version_19": "a83f77b41cc213d282805a5b518f15abbf96331599119f0ef4aca4be037add7b",
"schema_version_2": "e8e9ff32478df04fcddad10a34cba2e8bb1e67e7977b5bd6cdc4c31ec94282b4",
"schema_version_20": "6c4e9b2c5bccdc3243c239c390fb1caa5e15624e669b2c07e14c126f6d2e2cd6",
"schema_version_3": "a54745dbc1c51c000f74d4e5068f1e2f43e83309f023415b1749a47d5c1e0f12",
"schema_version_4": "216ea3a7d3e1704e40c797b5dc47456517c27dbb6ca98bf88812f4f63d74b5d9",
"schema_version_5": "46397e2f5f2c82116786127e9f6a403e975b14d2ca7b652a48cd1ba843e6a27c",

84
storage/entry.go
View file

@ -35,14 +35,41 @@ func (s *Storage) NewEntryQueryBuilder(userID int64) *EntryQueryBuilder {
return NewEntryQueryBuilder(s, userID)
}
// UpdateEntryContent updates entry content.
func (s *Storage) UpdateEntryContent(entry *model.Entry) error {
tx, err := s.db.Begin()
if err != nil {
return err
}
_, err = tx.Exec(`UPDATE entries SET content=$1 WHERE id=$2 AND user_id=$3`, entry.Content, entry.ID, entry.UserID)
if err != nil {
tx.Rollback()
return err
}
query := `
UPDATE entries
SET document_vectors = to_tsvector(title || ' ' || coalesce(content, ''))
WHERE id=$1 AND user_id=$2
`
_, err = tx.Exec(query, entry.ID, entry.UserID)
if err != nil {
tx.Rollback()
return err
}
return tx.Commit()
}
// createEntry add a new entry.
func (s *Storage) createEntry(entry *model.Entry) error {
query := `
INSERT INTO entries
(title, hash, url, comments_url, published_at, content, author, user_id, feed_id)
(title, hash, url, comments_url, published_at, content, author, user_id, feed_id, document_vectors)
VALUES
($1, $2, $3, $4, $5, $6, $7, $8, $9)
RETURNING id
($1, $2, $3, $4, $5, $6, $7, $8, $9, to_tsvector($1 || ' ' || coalesce($6, '')))
RETURNING id, status
`
err := s.db.QueryRow(
query,
@ -55,13 +82,12 @@ func (s *Storage) createEntry(entry *model.Entry) error {
entry.Author,
entry.UserID,
entry.FeedID,
).Scan(&entry.ID)
).Scan(&entry.ID, &entry.Status)
if err != nil {
return fmt.Errorf("unable to create entry: %v", err)
}
entry.Status = "unread"
for i := 0; i < len(entry.Enclosures); i++ {
entry.Enclosures[i].EntryID = entry.ID
entry.Enclosures[i].UserID = entry.UserID
@ -74,30 +100,14 @@ func (s *Storage) createEntry(entry *model.Entry) error {
return nil
}
// UpdateEntryContent updates entry content.
func (s *Storage) UpdateEntryContent(entry *model.Entry) error {
query := `
UPDATE entries SET
content=$1
WHERE user_id=$2 AND id=$3
`
_, err := s.db.Exec(
query,
entry.Content,
entry.UserID,
entry.ID,
)
return err
}
// updateEntry updates an entry when a feed is refreshed.
// Note: we do not update the published date because some feeds do not contains any date,
// it default to time.Now() which could change the order of items on the history page.
func (s *Storage) updateEntry(entry *model.Entry) error {
query := `
UPDATE entries SET
title=$1, url=$2, comments_url=$3, content=$4, author=$5
title=$1, url=$2, comments_url=$3, content=$4, author=$5,
document_vectors=to_tsvector($1 || ' ' || coalesce($4, ''))
WHERE user_id=$6 AND feed_id=$7 AND hash=$8
RETURNING id
`
@ -133,6 +143,20 @@ func (s *Storage) entryExists(entry *model.Entry) bool {
return result >= 1
}
// cleanupEntries deletes from the database entries marked as "removed" and not visible anymore in the feed.
func (s *Storage) cleanupEntries(feedID int64, entryHashes []string) error {
query := `
DELETE FROM entries
WHERE feed_id=$1 AND
id IN (SELECT id FROM entries WHERE feed_id=$2 AND status=$3 AND NOT (hash=ANY($4)))
`
if _, err := s.db.Exec(query, feedID, feedID, model.EntryStatusRemoved, pq.Array(entryHashes)); err != nil {
return fmt.Errorf("unable to cleanup entries: %v", err)
}
return nil
}
// UpdateEntries updates a list of entries while refreshing a feed.
func (s *Storage) UpdateEntries(userID, feedID int64, entries model.Entries, updateExistingEntries bool) (err error) {
var entryHashes []string
@ -162,20 +186,6 @@ func (s *Storage) UpdateEntries(userID, feedID int64, entries model.Entries, upd
return nil
}
// cleanupEntries deletes from the database entries marked as "removed" and not visible anymore in the feed.
func (s *Storage) cleanupEntries(feedID int64, entryHashes []string) error {
query := `
DELETE FROM entries
WHERE feed_id=$1 AND
id IN (SELECT id FROM entries WHERE feed_id=$2 AND status=$3 AND NOT (hash=ANY($4)))
`
if _, err := s.db.Exec(query, feedID, feedID, model.EntryStatusRemoved, pq.Array(entryHashes)); err != nil {
return fmt.Errorf("unable to cleanup entries: %v", err)
}
return nil
}
// ArchiveEntries changes the status of read items to "removed" after 60 days.
func (s *Storage) ArchiveEntries() error {
query := `

11
storage/entry_query_builder.go
View file

@ -27,6 +27,15 @@ type EntryQueryBuilder struct {
offset int
}
// WithSearchQuery adds full-text search query to the condition.
func (e *EntryQueryBuilder) WithSearchQuery(query string) *EntryQueryBuilder {
if query != "" {
e.conditions = append(e.conditions, fmt.Sprintf("e.document_vectors @@ plainto_tsquery($%d)", len(e.args)+1))
e.args = append(e.args, query)
}
return e
}
// WithStarred adds starred filter.
func (e *EntryQueryBuilder) WithStarred() *EntryQueryBuilder {
e.conditions = append(e.conditions, "e.starred is true")
@ -146,7 +155,7 @@ func (e *EntryQueryBuilder) CountEntries() (count int, err error) {
query := `SELECT count(*) FROM entries e LEFT JOIN feeds f ON f.id=e.feed_id WHERE %s`
condition := e.buildCondition()
defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[EntryQueryBuilder:CountEntries] condition=%s, args=%v", condition, e.args))
defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[EntryQueryBuilder:CountEntries] %s, args=%v", condition, e.args))
err = e.store.db.QueryRow(fmt.Sprintf(query, condition), e.args...).Scan(&count)
if err != nil {

2
storage/migration.go
View file

@ -12,7 +12,7 @@ import (
"github.com/miniflux/miniflux/sql"
)
const schemaVersion = 19
const schemaVersion = 20
// Migrate run database migrations.
func (s *Storage) Migrate() {