Add full-text search for entries and add search parameter to the API

api/entry.go

@@ -248,4 +248,9 @@ func configureFilters(builder *storage.EntryQueryBuilder, r *http.Request) {
 	if request.HasQueryParam(r, "starred") {
 		builder.WithStarred()
 	}
+
+	searchQuery := request.QueryParam(r, "search", "")
+	if searchQuery != "" {
+		builder.WithSearchQuery(searchQuery)
+	}
 }

locale/translations.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2018-06-30 23:41:46.05193077 +0200 CEST m=+0.307244685
+// 2018-07-04 14:42:27.494264089 -0700 PDT m=+0.053526807
 
 package locale
 
@@ -580,7 +580,7 @@ var translations = map[string]string{
     "Yes": "Ja",
     "No": "Nee",
     "This feed already exists (%s)": "Deze feed bestaat al (%s)",
-    "Unable to fetch feed (Status Code = %d)": "Kon feed niet update (code=%d)",
+    "Unable to fetch feed (Status Code = %d)": "Kon feed niet updaten (statuscode = %d)",
     "Unable to open this link: %v": "Kon link niet volgen: %v",
     "Unable to analyze this page: %v": "Kon pagina niet analyseren: %v",
     "Unable to find any subscription.": "Kon geen feeds vinden.",
@@ -650,7 +650,7 @@ var translations = map[string]string{
     "Scraper Rules": "Scraper regels",
     "Rewrite Rules": "Rewrite regels",
     "Preferences saved!": "Instellingen opgeslagen!",
-    "Your external account is now linked!": "Jouw externe account is nu gekoppeld!",
+    "Your external account is now linked!": "Je externe account is nu gekoppeld!",
     "Save articles to Wallabag": "Sauvegarder les articles vers Wallabag",
     "Wallabag API Endpoint": "Wallabag URL",
     "Wallabag Client ID": "Wallabag Client-ID",
@@ -1167,7 +1167,7 @@ var translationsChecksums = map[string]string{
 	"de_DE": "eddbb2c3224169a6533eed6f917af95b8d9bee58c3b3d61951260face7edd768",
 	"en_US": "6fe95384260941e8a5a3c695a655a932e0a8a6a572c1e45cb2b1ae8baa01b897",
 	"fr_FR": "7a451a1d09e051a847f554937e07a6af24b92f1da7f46da8c0ef2d4cc31bcec6",
-	"nl_NL": "bec647fdfb325a30050c50bdb9a29fee58b0705109d32c09b5ace77aa0777e7c",
+	"nl_NL": "05cca4936bd3b0fa44057c4dab64acdef3aed32fbb682393f254cfe2f686ef1f",
 	"pl_PL": "2295f35a98c8f60cfc6bab241d26b224c06979cc9ca3740bb89c63c7596a0431",
 	"zh_CN": "f5fb0a9b7336c51e74d727a2fb294bab3514e3002376da7fd904e0d7caed1a1c",
 }

sql/schema_version_20.sql

@@ -0,0 +1,3 @@
+alter table entries add column document_vectors tsvector;
+update entries set document_vectors = to_tsvector(title || ' ' || coalesce(content, ''));
+create index document_vectors_idx on entries using gin(document_vectors);

sql/sql.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2018-06-19 22:56:40.283528941 -0700 PDT m=+0.005341061
+// 2018-07-04 14:42:27.443758746 -0700 PDT m=+0.003021464
 
 package sql
 
@@ -142,6 +142,9 @@ alter table feeds add column password text default '';`,
 alter table users add column extra hstore;
 create index users_extra_idx on users using gin(extra);
 `,
+	"schema_version_20": `alter table entries add column document_vectors tsvector;
+update entries set document_vectors = to_tsvector(title || ' ' || coalesce(content, ''));
+create index document_vectors_idx on entries using gin(document_vectors);`,
 	"schema_version_3": `create table tokens (
     id text not null,
     value text not null,
@@ -189,6 +192,7 @@ var SqlMapChecksums = map[string]string{
 	"schema_version_18": "c0ec24847612c7f2dc326cf735baffba79391a56aedd73292371a39f38724a71",
 	"schema_version_19": "a83f77b41cc213d282805a5b518f15abbf96331599119f0ef4aca4be037add7b",
 	"schema_version_2":  "e8e9ff32478df04fcddad10a34cba2e8bb1e67e7977b5bd6cdc4c31ec94282b4",
+	"schema_version_20": "6c4e9b2c5bccdc3243c239c390fb1caa5e15624e669b2c07e14c126f6d2e2cd6",
 	"schema_version_3":  "a54745dbc1c51c000f74d4e5068f1e2f43e83309f023415b1749a47d5c1e0f12",
 	"schema_version_4":  "216ea3a7d3e1704e40c797b5dc47456517c27dbb6ca98bf88812f4f63d74b5d9",
 	"schema_version_5":  "46397e2f5f2c82116786127e9f6a403e975b14d2ca7b652a48cd1ba843e6a27c",

storage/entry.go

@@ -35,14 +35,41 @@ func (s *Storage) NewEntryQueryBuilder(userID int64) *EntryQueryBuilder {
 	return NewEntryQueryBuilder(s, userID)
 }
 
+// UpdateEntryContent updates entry content.
+func (s *Storage) UpdateEntryContent(entry *model.Entry) error {
+	tx, err := s.db.Begin()
+	if err != nil {
+		return err
+	}
+
+	_, err = tx.Exec(`UPDATE entries SET content=$1 WHERE id=$2 AND user_id=$3`, entry.Content, entry.ID, entry.UserID)
+	if err != nil {
+		tx.Rollback()
+		return err
+	}
+
+	query := `
+		UPDATE entries
+		SET document_vectors = to_tsvector(title || ' ' || coalesce(content, ''))
+		WHERE id=$1 AND user_id=$2
+	`
+	_, err = tx.Exec(query, entry.ID, entry.UserID)
+	if err != nil {
+		tx.Rollback()
+		return err
+	}
+
+	return tx.Commit()
+}
+
 // createEntry add a new entry.
 func (s *Storage) createEntry(entry *model.Entry) error {
 	query := `
 		INSERT INTO entries
-		(title, hash, url, comments_url, published_at, content, author, user_id, feed_id)
+		(title, hash, url, comments_url, published_at, content, author, user_id, feed_id, document_vectors)
 		VALUES
-		($1, $2, $3, $4, $5, $6, $7, $8, $9)
+		($1, $2, $3, $4, $5, $6, $7, $8, $9, to_tsvector($1 || ' ' || coalesce($6, '')))
-		RETURNING id
+		RETURNING id, status
 	`
 	err := s.db.QueryRow(
 		query,
@@ -55,13 +82,12 @@ func (s *Storage) createEntry(entry *model.Entry) error {
 		entry.Author,
 		entry.UserID,
 		entry.FeedID,
-	).Scan(&entry.ID)
+	).Scan(&entry.ID, &entry.Status)
 
 	if err != nil {
 		return fmt.Errorf("unable to create entry: %v", err)
 	}
 
-	entry.Status = "unread"
 	for i := 0; i < len(entry.Enclosures); i++ {
 		entry.Enclosures[i].EntryID = entry.ID
 		entry.Enclosures[i].UserID = entry.UserID
@@ -74,30 +100,14 @@ func (s *Storage) createEntry(entry *model.Entry) error {
 	return nil
 }
 
-// UpdateEntryContent updates entry content.
-func (s *Storage) UpdateEntryContent(entry *model.Entry) error {
-	query := `
-		UPDATE entries SET
-		content=$1
-		WHERE user_id=$2 AND id=$3
-	`
-
-	_, err := s.db.Exec(
-		query,
-		entry.Content,
-		entry.UserID,
-		entry.ID,
-	)
-	return err
-}
-
 // updateEntry updates an entry when a feed is refreshed.
 // Note: we do not update the published date because some feeds do not contains any date,
 // it default to time.Now() which could change the order of items on the history page.
 func (s *Storage) updateEntry(entry *model.Entry) error {
 	query := `
 		UPDATE entries SET
-		title=$1, url=$2, comments_url=$3, content=$4, author=$5
+		title=$1, url=$2, comments_url=$3, content=$4, author=$5,
+		document_vectors=to_tsvector($1 || ' ' || coalesce($4, ''))
 		WHERE user_id=$6 AND feed_id=$7 AND hash=$8
 		RETURNING id
 	`
@@ -133,6 +143,20 @@ func (s *Storage) entryExists(entry *model.Entry) bool {
 	return result >= 1
 }
 
+// cleanupEntries deletes from the database entries marked as "removed" and not visible anymore in the feed.
+func (s *Storage) cleanupEntries(feedID int64, entryHashes []string) error {
+	query := `
+		DELETE FROM entries
+		WHERE feed_id=$1 AND
+		id IN (SELECT id FROM entries WHERE feed_id=$2 AND status=$3 AND NOT (hash=ANY($4)))
+	`
+	if _, err := s.db.Exec(query, feedID, feedID, model.EntryStatusRemoved, pq.Array(entryHashes)); err != nil {
+		return fmt.Errorf("unable to cleanup entries: %v", err)
+	}
+
+	return nil
+}
+
 // UpdateEntries updates a list of entries while refreshing a feed.
 func (s *Storage) UpdateEntries(userID, feedID int64, entries model.Entries, updateExistingEntries bool) (err error) {
 	var entryHashes []string
@@ -162,20 +186,6 @@ func (s *Storage) UpdateEntries(userID, feedID int64, entries model.Entries, upd
 	return nil
 }
 
-// cleanupEntries deletes from the database entries marked as "removed" and not visible anymore in the feed.
-func (s *Storage) cleanupEntries(feedID int64, entryHashes []string) error {
-	query := `
-		DELETE FROM entries
-		WHERE feed_id=$1 AND
-		id IN (SELECT id FROM entries WHERE feed_id=$2 AND status=$3 AND NOT (hash=ANY($4)))
-	`
-	if _, err := s.db.Exec(query, feedID, feedID, model.EntryStatusRemoved, pq.Array(entryHashes)); err != nil {
-		return fmt.Errorf("unable to cleanup entries: %v", err)
-	}
-
-	return nil
-}
-
 // ArchiveEntries changes the status of read items to "removed" after 60 days.
 func (s *Storage) ArchiveEntries() error {
 	query := `

storage/entry_query_builder.go

@@ -27,6 +27,15 @@ type EntryQueryBuilder struct {
 	offset     int
 }
 
+// WithSearchQuery adds full-text search query to the condition.
+func (e *EntryQueryBuilder) WithSearchQuery(query string) *EntryQueryBuilder {
+	if query != "" {
+		e.conditions = append(e.conditions, fmt.Sprintf("e.document_vectors @@ plainto_tsquery($%d)", len(e.args)+1))
+		e.args = append(e.args, query)
+	}
+	return e
+}
+
 // WithStarred adds starred filter.
 func (e *EntryQueryBuilder) WithStarred() *EntryQueryBuilder {
 	e.conditions = append(e.conditions, "e.starred is true")
@@ -146,7 +155,7 @@ func (e *EntryQueryBuilder) CountEntries() (count int, err error) {
 	query := `SELECT count(*) FROM entries e LEFT JOIN feeds f ON f.id=e.feed_id WHERE %s`
 	condition := e.buildCondition()
 
-	defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[EntryQueryBuilder:CountEntries] condition=%s, args=%v", condition, e.args))
+	defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[EntryQueryBuilder:CountEntries] %s, args=%v", condition, e.args))
 
 	err = e.store.db.QueryRow(fmt.Sprintf(query, condition), e.args...).Scan(&count)
 	if err != nil {

storage/migration.go

@@ -12,7 +12,7 @@ import (
 	"github.com/miniflux/miniflux/sql"
 )
 
-const schemaVersion = 19
+const schemaVersion = 20
 
 // Migrate run database migrations.
 func (s *Storage) Migrate() {