From af15412954b9b36365f3fd723bf91b3c7c2f88bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Guillot?= Date: Wed, 4 Jul 2018 17:40:03 -0700 Subject: [PATCH] Add full-text search for entries and add search parameter to the API --- api/entry.go | 5 ++ locale/translations.go | 8 ++-- sql/schema_version_20.sql | 3 ++ sql/sql.go | 6 ++- storage/entry.go | 84 +++++++++++++++++++--------------- storage/entry_query_builder.go | 11 ++++- storage/migration.go | 2 +- 7 files changed, 75 insertions(+), 44 deletions(-) create mode 100644 sql/schema_version_20.sql diff --git a/api/entry.go b/api/entry.go index 6d1167b5..a6785fbb 100644 --- a/api/entry.go +++ b/api/entry.go @@ -248,4 +248,9 @@ func configureFilters(builder *storage.EntryQueryBuilder, r *http.Request) { if request.HasQueryParam(r, "starred") { builder.WithStarred() } + + searchQuery := request.QueryParam(r, "search", "") + if searchQuery != "" { + builder.WithSearchQuery(searchQuery) + } } diff --git a/locale/translations.go b/locale/translations.go index 34891194..aff5b7c8 100755 --- a/locale/translations.go +++ b/locale/translations.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2018-06-30 23:41:46.05193077 +0200 CEST m=+0.307244685 +// 2018-07-04 14:42:27.494264089 -0700 PDT m=+0.053526807 package locale @@ -580,7 +580,7 @@ var translations = map[string]string{ "Yes": "Ja", "No": "Nee", "This feed already exists (%s)": "Deze feed bestaat al (%s)", - "Unable to fetch feed (Status Code = %d)": "Kon feed niet update (code=%d)", + "Unable to fetch feed (Status Code = %d)": "Kon feed niet updaten (statuscode = %d)", "Unable to open this link: %v": "Kon link niet volgen: %v", "Unable to analyze this page: %v": "Kon pagina niet analyseren: %v", "Unable to find any subscription.": "Kon geen feeds vinden.", @@ -650,7 +650,7 @@ var translations = map[string]string{ "Scraper Rules": "Scraper regels", "Rewrite Rules": "Rewrite regels", "Preferences saved!": "Instellingen opgeslagen!", - "Your external account is now linked!": "Jouw externe account is nu gekoppeld!", + "Your external account is now linked!": "Je externe account is nu gekoppeld!", "Save articles to Wallabag": "Sauvegarder les articles vers Wallabag", "Wallabag API Endpoint": "Wallabag URL", "Wallabag Client ID": "Wallabag Client-ID", @@ -1167,7 +1167,7 @@ var translationsChecksums = map[string]string{ "de_DE": "eddbb2c3224169a6533eed6f917af95b8d9bee58c3b3d61951260face7edd768", "en_US": "6fe95384260941e8a5a3c695a655a932e0a8a6a572c1e45cb2b1ae8baa01b897", "fr_FR": "7a451a1d09e051a847f554937e07a6af24b92f1da7f46da8c0ef2d4cc31bcec6", - "nl_NL": "bec647fdfb325a30050c50bdb9a29fee58b0705109d32c09b5ace77aa0777e7c", + "nl_NL": "05cca4936bd3b0fa44057c4dab64acdef3aed32fbb682393f254cfe2f686ef1f", "pl_PL": "2295f35a98c8f60cfc6bab241d26b224c06979cc9ca3740bb89c63c7596a0431", "zh_CN": "f5fb0a9b7336c51e74d727a2fb294bab3514e3002376da7fd904e0d7caed1a1c", } diff --git a/sql/schema_version_20.sql b/sql/schema_version_20.sql new file mode 100644 index 00000000..228e1624 --- /dev/null +++ b/sql/schema_version_20.sql @@ -0,0 +1,3 @@ +alter table entries add column document_vectors tsvector; +update entries set document_vectors = to_tsvector(title || ' ' || coalesce(content, '')); +create index document_vectors_idx on entries using gin(document_vectors); \ No newline at end of file diff --git a/sql/sql.go b/sql/sql.go index d0958938..fae19aac 100644 --- a/sql/sql.go +++ b/sql/sql.go @@ -1,5 +1,5 @@ // Code generated by go generate; DO NOT EDIT. -// 2018-06-19 22:56:40.283528941 -0700 PDT m=+0.005341061 +// 2018-07-04 14:42:27.443758746 -0700 PDT m=+0.003021464 package sql @@ -142,6 +142,9 @@ alter table feeds add column password text default '';`, alter table users add column extra hstore; create index users_extra_idx on users using gin(extra); `, + "schema_version_20": `alter table entries add column document_vectors tsvector; +update entries set document_vectors = to_tsvector(title || ' ' || coalesce(content, '')); +create index document_vectors_idx on entries using gin(document_vectors);`, "schema_version_3": `create table tokens ( id text not null, value text not null, @@ -189,6 +192,7 @@ var SqlMapChecksums = map[string]string{ "schema_version_18": "c0ec24847612c7f2dc326cf735baffba79391a56aedd73292371a39f38724a71", "schema_version_19": "a83f77b41cc213d282805a5b518f15abbf96331599119f0ef4aca4be037add7b", "schema_version_2": "e8e9ff32478df04fcddad10a34cba2e8bb1e67e7977b5bd6cdc4c31ec94282b4", + "schema_version_20": "6c4e9b2c5bccdc3243c239c390fb1caa5e15624e669b2c07e14c126f6d2e2cd6", "schema_version_3": "a54745dbc1c51c000f74d4e5068f1e2f43e83309f023415b1749a47d5c1e0f12", "schema_version_4": "216ea3a7d3e1704e40c797b5dc47456517c27dbb6ca98bf88812f4f63d74b5d9", "schema_version_5": "46397e2f5f2c82116786127e9f6a403e975b14d2ca7b652a48cd1ba843e6a27c", diff --git a/storage/entry.go b/storage/entry.go index 65c02200..b703a4ce 100644 --- a/storage/entry.go +++ b/storage/entry.go @@ -35,14 +35,41 @@ func (s *Storage) NewEntryQueryBuilder(userID int64) *EntryQueryBuilder { return NewEntryQueryBuilder(s, userID) } +// UpdateEntryContent updates entry content. +func (s *Storage) UpdateEntryContent(entry *model.Entry) error { + tx, err := s.db.Begin() + if err != nil { + return err + } + + _, err = tx.Exec(`UPDATE entries SET content=$1 WHERE id=$2 AND user_id=$3`, entry.Content, entry.ID, entry.UserID) + if err != nil { + tx.Rollback() + return err + } + + query := ` + UPDATE entries + SET document_vectors = to_tsvector(title || ' ' || coalesce(content, '')) + WHERE id=$1 AND user_id=$2 + ` + _, err = tx.Exec(query, entry.ID, entry.UserID) + if err != nil { + tx.Rollback() + return err + } + + return tx.Commit() +} + // createEntry add a new entry. func (s *Storage) createEntry(entry *model.Entry) error { query := ` INSERT INTO entries - (title, hash, url, comments_url, published_at, content, author, user_id, feed_id) + (title, hash, url, comments_url, published_at, content, author, user_id, feed_id, document_vectors) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9) - RETURNING id + ($1, $2, $3, $4, $5, $6, $7, $8, $9, to_tsvector($1 || ' ' || coalesce($6, ''))) + RETURNING id, status ` err := s.db.QueryRow( query, @@ -55,13 +82,12 @@ func (s *Storage) createEntry(entry *model.Entry) error { entry.Author, entry.UserID, entry.FeedID, - ).Scan(&entry.ID) + ).Scan(&entry.ID, &entry.Status) if err != nil { return fmt.Errorf("unable to create entry: %v", err) } - entry.Status = "unread" for i := 0; i < len(entry.Enclosures); i++ { entry.Enclosures[i].EntryID = entry.ID entry.Enclosures[i].UserID = entry.UserID @@ -74,30 +100,14 @@ func (s *Storage) createEntry(entry *model.Entry) error { return nil } -// UpdateEntryContent updates entry content. -func (s *Storage) UpdateEntryContent(entry *model.Entry) error { - query := ` - UPDATE entries SET - content=$1 - WHERE user_id=$2 AND id=$3 - ` - - _, err := s.db.Exec( - query, - entry.Content, - entry.UserID, - entry.ID, - ) - return err -} - // updateEntry updates an entry when a feed is refreshed. // Note: we do not update the published date because some feeds do not contains any date, // it default to time.Now() which could change the order of items on the history page. func (s *Storage) updateEntry(entry *model.Entry) error { query := ` UPDATE entries SET - title=$1, url=$2, comments_url=$3, content=$4, author=$5 + title=$1, url=$2, comments_url=$3, content=$4, author=$5, + document_vectors=to_tsvector($1 || ' ' || coalesce($4, '')) WHERE user_id=$6 AND feed_id=$7 AND hash=$8 RETURNING id ` @@ -133,6 +143,20 @@ func (s *Storage) entryExists(entry *model.Entry) bool { return result >= 1 } +// cleanupEntries deletes from the database entries marked as "removed" and not visible anymore in the feed. +func (s *Storage) cleanupEntries(feedID int64, entryHashes []string) error { + query := ` + DELETE FROM entries + WHERE feed_id=$1 AND + id IN (SELECT id FROM entries WHERE feed_id=$2 AND status=$3 AND NOT (hash=ANY($4))) + ` + if _, err := s.db.Exec(query, feedID, feedID, model.EntryStatusRemoved, pq.Array(entryHashes)); err != nil { + return fmt.Errorf("unable to cleanup entries: %v", err) + } + + return nil +} + // UpdateEntries updates a list of entries while refreshing a feed. func (s *Storage) UpdateEntries(userID, feedID int64, entries model.Entries, updateExistingEntries bool) (err error) { var entryHashes []string @@ -162,20 +186,6 @@ func (s *Storage) UpdateEntries(userID, feedID int64, entries model.Entries, upd return nil } -// cleanupEntries deletes from the database entries marked as "removed" and not visible anymore in the feed. -func (s *Storage) cleanupEntries(feedID int64, entryHashes []string) error { - query := ` - DELETE FROM entries - WHERE feed_id=$1 AND - id IN (SELECT id FROM entries WHERE feed_id=$2 AND status=$3 AND NOT (hash=ANY($4))) - ` - if _, err := s.db.Exec(query, feedID, feedID, model.EntryStatusRemoved, pq.Array(entryHashes)); err != nil { - return fmt.Errorf("unable to cleanup entries: %v", err) - } - - return nil -} - // ArchiveEntries changes the status of read items to "removed" after 60 days. func (s *Storage) ArchiveEntries() error { query := ` diff --git a/storage/entry_query_builder.go b/storage/entry_query_builder.go index 05771921..8c0d706e 100644 --- a/storage/entry_query_builder.go +++ b/storage/entry_query_builder.go @@ -27,6 +27,15 @@ type EntryQueryBuilder struct { offset int } +// WithSearchQuery adds full-text search query to the condition. +func (e *EntryQueryBuilder) WithSearchQuery(query string) *EntryQueryBuilder { + if query != "" { + e.conditions = append(e.conditions, fmt.Sprintf("e.document_vectors @@ plainto_tsquery($%d)", len(e.args)+1)) + e.args = append(e.args, query) + } + return e +} + // WithStarred adds starred filter. func (e *EntryQueryBuilder) WithStarred() *EntryQueryBuilder { e.conditions = append(e.conditions, "e.starred is true") @@ -146,7 +155,7 @@ func (e *EntryQueryBuilder) CountEntries() (count int, err error) { query := `SELECT count(*) FROM entries e LEFT JOIN feeds f ON f.id=e.feed_id WHERE %s` condition := e.buildCondition() - defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[EntryQueryBuilder:CountEntries] condition=%s, args=%v", condition, e.args)) + defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[EntryQueryBuilder:CountEntries] %s, args=%v", condition, e.args)) err = e.store.db.QueryRow(fmt.Sprintf(query, condition), e.args...).Scan(&count) if err != nil { diff --git a/storage/migration.go b/storage/migration.go index 56b7bfb8..e78f1d55 100644 --- a/storage/migration.go +++ b/storage/migration.go @@ -12,7 +12,7 @@ import ( "github.com/miniflux/miniflux/sql" ) -const schemaVersion = 19 +const schemaVersion = 20 // Migrate run database migrations. func (s *Storage) Migrate() {