From 2659883ce5be7de27589b43de5f64e4068df535b Mon Sep 17 00:00:00 2001 From: Carsten <105346+MoriTanosuke@users.noreply.github.com> Date: Tue, 12 Jul 2022 06:12:26 +0200 Subject: [PATCH] Add rewrite rules for article URL before fetching content --- database/migrations.go | 6 +++ locale/translations/de_DE.json | 1 + locale/translations/el_EL.json | 1 + locale/translations/en_US.json | 1 + locale/translations/es_ES.json | 1 + locale/translations/fi_FI.json | 1 + locale/translations/fr_FR.json | 1 + locale/translations/it_IT.json | 1 + locale/translations/ja_JP.json | 1 + locale/translations/nl_NL.json | 1 + locale/translations/pl_PL.json | 1 + locale/translations/pt_BR.json | 1 + locale/translations/ru_RU.json | 1 + locale/translations/tr_TR.json | 1 + locale/translations/zh_CN.json | 1 + locale/translations/zh_TW.json | 1 + model/feed.go | 7 ++++ reader/handler/handler.go | 1 + reader/processor/processor.go | 38 ++++++++++++++----- storage/feed.go | 12 ++++-- storage/feed_query_builder.go | 2 + .../templates/views/add_subscription.html | 3 ++ .../templates/views/choose_subscription.html | 1 + template/templates/views/edit_feed.html | 7 +++- ui/feed_edit.go | 1 + ui/feed_update.go | 13 ++++--- ui/form/feed.go | 3 ++ ui/form/subscription.go | 6 +++ ui/subscription_choose.go | 1 + ui/subscription_submit.go | 1 + 30 files changed, 96 insertions(+), 21 deletions(-) diff --git a/database/migrations.go b/database/migrations.go index 47e9c7dc..7e68682f 100644 --- a/database/migrations.go +++ b/database/migrations.go @@ -591,4 +591,10 @@ var migrations = []func(tx *sql.Tx) error{ _, err = tx.Exec(sql) return err }, + func(tx *sql.Tx) (err error) { + _, err = tx.Exec(` + ALTER TABLE feeds ADD COLUMN url_rewrite_rules text not null default '' + `) + return err + }, } diff --git a/locale/translations/de_DE.json b/locale/translations/de_DE.json index 4b4ab61a..3b7afb13 100644 --- a/locale/translations/de_DE.json +++ b/locale/translations/de_DE.json @@ -275,6 +275,7 @@ "form.feed.label.rewrite_rules": "Umschreiberegeln", "form.feed.label.blocklist_rules": "Blockierregeln", "form.feed.label.keeplist_rules": "Erlaubnisregeln", + "form.feed.label.urlrewrite_rules": "Umschreibregeln für URL", "form.feed.label.ignore_http_cache": "Ignoriere HTTP-cache", "form.feed.label.allow_self_signed_certificates": "Erlaube selbstsignierte oder ungültige Zertifikate", "form.feed.label.fetch_via_proxy": "Über Proxy abrufen", diff --git a/locale/translations/el_EL.json b/locale/translations/el_EL.json index d1b0acec..f17a2f5b 100644 --- a/locale/translations/el_EL.json +++ b/locale/translations/el_EL.json @@ -259,6 +259,7 @@ "error.feed_category_not_found": "Αυτή η κατηγορία δεν υπάρχει ή δεν ανήκει σε αυτόν τον χρήστη.", "error.feed_invalid_blocklist_rule": "Ο κανόνας λίστας μπλοκ δεν είναι έγκυρος.", "error.feed_invalid_keeplist_rule": "Ο κανόνας keep list δεν είναι έγκυρος.", + "form.feed.label.urlrewrite_rules": "επανεγγραφή κανόνων για τη διεύθυνση URL.", "error.user_mandatory_fields": "Το όνομα χρήστη είναι υποχρεωτικό.", "error.api_key_already_exists": "Αυτό το κλειδί API υπάρχει ήδη.", "error.unable_to_create_api_key": "Δεν είναι δυνατή η δημιουργία αυτού του κλειδιού API.", diff --git a/locale/translations/en_US.json b/locale/translations/en_US.json index 4b2c8cb2..d6aacce4 100644 --- a/locale/translations/en_US.json +++ b/locale/translations/en_US.json @@ -275,6 +275,7 @@ "form.feed.label.rewrite_rules": "Rewrite Rules", "form.feed.label.blocklist_rules": "Block Rules", "form.feed.label.keeplist_rules": "Keep Rules", + "form.feed.label.urlrewrite_rules": "URL Rewrite Rules", "form.feed.label.ignore_http_cache": "Ignore HTTP cache", "form.feed.label.allow_self_signed_certificates": "Allow self-signed or invalid certificates", "form.feed.label.fetch_via_proxy": "Fetch via proxy", diff --git a/locale/translations/es_ES.json b/locale/translations/es_ES.json index cb57c149..38a13444 100644 --- a/locale/translations/es_ES.json +++ b/locale/translations/es_ES.json @@ -275,6 +275,7 @@ "form.feed.label.rewrite_rules": "Reglas de reescribir", "form.feed.label.blocklist_rules": "Reglas de Filtrado(Bloquear)", "form.feed.label.keeplist_rules": "Reglas de Filtrado(Permitir)", + "form.feed.label.urlrewrite_rules": "Reglas de Filtrado(reescritura)", "form.feed.label.ignore_http_cache": "Ignorar caché HTTP", "form.feed.label.allow_self_signed_certificates": "Permitir certificados autofirmados o no válidos", "form.feed.label.fetch_via_proxy": "Buscar a través de proxy", diff --git a/locale/translations/fi_FI.json b/locale/translations/fi_FI.json index f869adfa..89d74d14 100644 --- a/locale/translations/fi_FI.json +++ b/locale/translations/fi_FI.json @@ -259,6 +259,7 @@ "error.feed_category_not_found": "Tätä kategoriaa ei ole olemassa tai se ei kuulu tälle käyttäjälle.", "error.feed_invalid_blocklist_rule": "The block list rule is invalid.", "error.feed_invalid_keeplist_rule": "The keep list rule is invalid.", + "form.feed.label.urlrewrite_rules": "URL-osoitteen uudelleenkirjoitussäännöt", "error.user_mandatory_fields": "Käyttäjätunnus on pakollinen.", "error.api_key_already_exists": "API-avain on jo olemassa.", "error.unable_to_create_api_key": "API-avainta ei voi luoda.", diff --git a/locale/translations/fr_FR.json b/locale/translations/fr_FR.json index fd9c0a3b..505f9e8e 100644 --- a/locale/translations/fr_FR.json +++ b/locale/translations/fr_FR.json @@ -275,6 +275,7 @@ "form.feed.label.rewrite_rules": "Règles de réécriture", "form.feed.label.blocklist_rules": "Règles de blocage", "form.feed.label.keeplist_rules": "Règles d'autorisation", + "form.feed.label.urlrewrite_rules": "Règles de réécriture d'URL", "form.feed.label.ignore_http_cache": "Ignorer le cache HTTP", "form.feed.label.allow_self_signed_certificates": "Autoriser les certificats auto-signés ou non valides", "form.feed.label.fetch_via_proxy": "Récupérer via proxy", diff --git a/locale/translations/it_IT.json b/locale/translations/it_IT.json index 10c36ed8..e46249c4 100644 --- a/locale/translations/it_IT.json +++ b/locale/translations/it_IT.json @@ -275,6 +275,7 @@ "form.feed.label.rewrite_rules": "Regole di impaginazione del contenuto", "form.feed.label.blocklist_rules": "Regole di blocco", "form.feed.label.keeplist_rules": "Regole di autorizzazione", + "form.feed.label.urlrewrite_rules": "Regole di riscrittura URL", "form.feed.label.ignore_http_cache": "Ignora cache HTTP", "form.feed.label.allow_self_signed_certificates": "Consenti certificati autofirmati o non validi", "form.feed.label.fetch_via_proxy": "Recuperare tramite proxy", diff --git a/locale/translations/ja_JP.json b/locale/translations/ja_JP.json index 29bcef53..1535c3ee 100644 --- a/locale/translations/ja_JP.json +++ b/locale/translations/ja_JP.json @@ -275,6 +275,7 @@ "form.feed.label.rewrite_rules": "Rewrite ルール", "form.feed.label.blocklist_rules": "ブロックルール", "form.feed.label.keeplist_rules": "許可規則", + "form.feed.label.urlrewrite_rules": "URL書き換えルール", "form.feed.label.ignore_http_cache": "HTTPキャッシュを無視", "form.feed.label.allow_self_signed_certificates": "自己署名証明書または無効な証明書を許可する", "form.feed.label.fetch_via_proxy": "プロキシ経由でフェッチ", diff --git a/locale/translations/nl_NL.json b/locale/translations/nl_NL.json index 638370b3..291cf85d 100644 --- a/locale/translations/nl_NL.json +++ b/locale/translations/nl_NL.json @@ -275,6 +275,7 @@ "form.feed.label.rewrite_rules": "Rewrite regels", "form.feed.label.blocklist_rules": "Blokkeer regels", "form.feed.label.keeplist_rules": "toestemmingsregels", + "form.feed.label.urlrewrite_rules": "Regels voor het herschrijven van URL's", "form.feed.label.ignore_http_cache": "Negeer HTTP-cache", "form.feed.label.allow_self_signed_certificates": "Sta zelfondertekende of ongeldige certificaten toe", "form.feed.label.fetch_via_proxy": "Ophalen via proxy", diff --git a/locale/translations/pl_PL.json b/locale/translations/pl_PL.json index d3f08e3f..28abe75b 100644 --- a/locale/translations/pl_PL.json +++ b/locale/translations/pl_PL.json @@ -277,6 +277,7 @@ "form.feed.label.rewrite_rules": "Reguły zapisu", "form.feed.label.blocklist_rules": "Zasady blokowania", "form.feed.label.keeplist_rules": "Zasady zezwoleń", + "form.feed.label.urlrewrite_rules": "Zasady przepisywania adresów URL", "form.feed.label.ignore_http_cache": "Zignoruj ​​pamięć podręczną HTTP", "form.feed.label.allow_self_signed_certificates": "Zezwalaj na certyfikaty z podpisem własnym lub nieprawidłowe certyfikaty", "form.feed.label.fetch_via_proxy": "Pobierz przez proxy", diff --git a/locale/translations/pt_BR.json b/locale/translations/pt_BR.json index 6a93ea62..15e338fd 100644 --- a/locale/translations/pt_BR.json +++ b/locale/translations/pt_BR.json @@ -275,6 +275,7 @@ "form.feed.label.rewrite_rules": "Regras para o Rewrite", "form.feed.label.blocklist_rules": "Regras de bloqueio", "form.feed.label.keeplist_rules": "Regras de permissão", + "form.feed.label.urlrewrite_rules": "Regras de reescrita de URL", "form.feed.label.ignore_http_cache": "Ignorar cache HTTP", "form.feed.label.allow_self_signed_certificates": "Permitir certificados autoassinados ou inválidos", "form.feed.label.disabled": "Não atualizar esta fonte", diff --git a/locale/translations/ru_RU.json b/locale/translations/ru_RU.json index ec54657c..4fe38f5f 100644 --- a/locale/translations/ru_RU.json +++ b/locale/translations/ru_RU.json @@ -277,6 +277,7 @@ "form.feed.label.rewrite_rules": "Правила Rewrite", "form.feed.label.blocklist_rules": "Правила блокировки", "form.feed.label.keeplist_rules": "правила разрешений", + "form.feed.label.urlrewrite_rules": "Правила перезаписи URL", "form.feed.label.ignore_http_cache": "Игнорировать HTTP-кеш", "form.feed.label.allow_self_signed_certificates": "Разрешить самоподписанные или недействительные сертификаты", "form.feed.label.fetch_via_proxy": "Получить через прокси", diff --git a/locale/translations/tr_TR.json b/locale/translations/tr_TR.json index d7e6c8f6..6a3abeff 100644 --- a/locale/translations/tr_TR.json +++ b/locale/translations/tr_TR.json @@ -275,6 +275,7 @@ "form.feed.label.rewrite_rules": "Yeniden Yazma Kuralları", "form.feed.label.blocklist_rules": "Engelleme Kuralları", "form.feed.label.keeplist_rules": "Saklama Kuralları", + "form.feed.label.urlrewrite_rules": "URL Yeniden Yazma Kuralları", "form.feed.label.ignore_http_cache": "HTTP önbelleğini yoksay", "form.feed.label.allow_self_signed_certificates": "Kendinden imzalı veya geçersiz sertifikalara izin ver", "form.feed.label.fetch_via_proxy": "Proxy ile çek", diff --git a/locale/translations/zh_CN.json b/locale/translations/zh_CN.json index 877b298d..1ead7e4e 100644 --- a/locale/translations/zh_CN.json +++ b/locale/translations/zh_CN.json @@ -273,6 +273,7 @@ "form.feed.label.rewrite_rules": "重写规则", "form.feed.label.blocklist_rules": "阻止规则", "form.feed.label.keeplist_rules": "保留规则", + "form.feed.label.urlrewrite_rules": "URL 重写规则", "form.feed.label.ignore_http_cache": "忽略 HTTP 缓存", "form.feed.label.allow_self_signed_certificates": "允许自签名证书或无效证书", "form.feed.label.fetch_via_proxy": "通过代理获取", diff --git a/locale/translations/zh_TW.json b/locale/translations/zh_TW.json index 59e19847..12f9a3db 100644 --- a/locale/translations/zh_TW.json +++ b/locale/translations/zh_TW.json @@ -275,6 +275,7 @@ "form.feed.label.rewrite_rules": "重寫規則", "form.feed.label.blocklist_rules": "過濾規則", "form.feed.label.keeplist_rules": "保留規則", + "form.feed.label.urlrewrite_rules": "URL 重写规则", "form.feed.label.ignore_http_cache": "忽略 HTTP 快取", "form.feed.label.allow_self_signed_certificates": "允許自簽章憑證或無效憑證", "form.feed.label.fetch_via_proxy": "透過代理獲取", diff --git a/model/feed.go b/model/feed.go index f515988f..7068404f 100644 --- a/model/feed.go +++ b/model/feed.go @@ -40,6 +40,7 @@ type Feed struct { Crawler bool `json:"crawler"` BlocklistRules string `json:"blocklist_rules"` KeeplistRules string `json:"keeplist_rules"` + UrlRewriteRules string `json:"urlrewrite_rules"` UserAgent string `json:"user_agent"` Cookie string `json:"cookie"` Username string `json:"username"` @@ -141,6 +142,7 @@ type FeedCreationRequest struct { BlocklistRules string `json:"blocklist_rules"` KeeplistRules string `json:"keeplist_rules"` HideGlobally bool `json:"hide_globally"` + UrlRewriteRules string `json:"urlrewrite_rules"` } // FeedModificationRequest represents the request to update a feed. @@ -152,6 +154,7 @@ type FeedModificationRequest struct { RewriteRules *string `json:"rewrite_rules"` BlocklistRules *string `json:"blocklist_rules"` KeeplistRules *string `json:"keeplist_rules"` + UrlRewriteRules *string `json:"urlrewrite_rules"` Crawler *bool `json:"crawler"` UserAgent *string `json:"user_agent"` Cookie *string `json:"cookie"` @@ -191,6 +194,10 @@ func (f *FeedModificationRequest) Patch(feed *Feed) { feed.KeeplistRules = *f.KeeplistRules } + if f.UrlRewriteRules != nil { + feed.UrlRewriteRules = *f.UrlRewriteRules + } + if f.BlocklistRules != nil { feed.BlocklistRules = *f.BlocklistRules } diff --git a/reader/handler/handler.go b/reader/handler/handler.go index 13f851d5..bd775104 100644 --- a/reader/handler/handler.go +++ b/reader/handler/handler.go @@ -74,6 +74,7 @@ func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model subscription.RewriteRules = feedCreationRequest.RewriteRules subscription.BlocklistRules = feedCreationRequest.BlocklistRules subscription.KeeplistRules = feedCreationRequest.KeeplistRules + subscription.UrlRewriteRules = feedCreationRequest.UrlRewriteRules subscription.WithCategoryID(feedCreationRequest.CategoryID) subscription.WithClientResponse(response) subscription.CheckedNow() diff --git a/reader/processor/processor.go b/reader/processor/processor.go index a2165f51..4df0e510 100644 --- a/reader/processor/processor.go +++ b/reader/processor/processor.go @@ -32,8 +32,9 @@ import ( ) var ( - youtubeRegex = regexp.MustCompile(`youtube\.com/watch\?v=(.*)`) - iso8601Regex = regexp.MustCompile(`^P((?P\d+)Y)?((?P\d+)M)?((?P\d+)W)?((?P\d+)D)?(T((?P\d+)H)?((?P\d+)M)?((?P\d+)S)?)?$`) + youtubeRegex = regexp.MustCompile(`youtube\.com/watch\?v=(.*)`) + iso8601Regex = regexp.MustCompile(`^P((?P\d+)Y)?((?P\d+)M)?((?P\d+)W)?((?P\d+)D)?(T((?P\d+)H)?((?P\d+)M)?((?P\d+)S)?)?$`) + customReplaceRuleRegex = regexp.MustCompile(`rewrite\("(.*)"\|"(.*)"\)`) ) // ProcessFeedEntries downloads original web page for entries and apply filters. @@ -47,13 +48,14 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed) { continue } + url := getUrlFromEntry(feed, entry) entryIsNew := !store.EntryURLExists(feed.ID, entry.URL) if feed.Crawler && entryIsNew { - logger.Debug("[Processor] Crawling entry %q from feed %q", entry.URL, feed.FeedURL) + logger.Debug("[Processor] Crawling entry %q from feed %q", url, feed.FeedURL) startTime := time.Now() content, scraperErr := scraper.Fetch( - entry.URL, + url, feed.ScraperRules, feed.UserAgent, feed.Cookie, @@ -77,10 +79,10 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed) { } } - entry.Content = rewrite.Rewriter(entry.URL, entry.Content, feed.RewriteRules) + entry.Content = rewrite.Rewriter(url, entry.Content, feed.RewriteRules) // The sanitizer should always run at the end of the process to make sure unsafe HTML is filtered. - entry.Content = sanitizer.Sanitize(entry.URL, entry.Content) + entry.Content = sanitizer.Sanitize(url, entry.Content) if entryIsNew { intg, err := store.Integration(feed.UserID) @@ -127,8 +129,10 @@ func isAllowedEntry(feed *model.Feed, entry *model.Entry) bool { // ProcessEntryWebPage downloads the entry web page and apply rewrite rules. func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry) error { startTime := time.Now() + url := getUrlFromEntry(feed, entry) + content, scraperErr := scraper.Fetch( - entry.URL, + url, entry.Feed.ScraperRules, entry.Feed.UserAgent, entry.Feed.Cookie, @@ -148,8 +152,8 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry) error { return scraperErr } - content = rewrite.Rewriter(entry.URL, content, entry.Feed.RewriteRules) - content = sanitizer.Sanitize(entry.URL, content) + content = rewrite.Rewriter(url, content, entry.Feed.RewriteRules) + content = sanitizer.Sanitize(url, content) if content != "" { entry.Content = content @@ -159,6 +163,22 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry) error { return nil } +func getUrlFromEntry(feed *model.Feed, entry *model.Entry) string { + var url = entry.URL + if feed.UrlRewriteRules != "" { + parts := customReplaceRuleRegex.FindStringSubmatch(feed.UrlRewriteRules) + + if len(parts) >= 3 { + re := regexp.MustCompile(parts[1]) + url = re.ReplaceAllString(entry.URL, parts[2]) + logger.Debug(`[Processor] Rewriting entry URL %s to %s`, entry.URL, url) + } else { + logger.Debug("[Processor] Cannot find search and replace terms for replace rule %s", feed.UrlRewriteRules) + } + } + return url +} + func updateEntryReadingTime(store *storage.Storage, feed *model.Feed, entry *model.Entry, entryIsNew bool) { if shouldFetchYouTubeWatchTime(entry) { if entryIsNew { diff --git a/storage/feed.go b/storage/feed.go index 0c662950..03d0ba1c 100644 --- a/storage/feed.go +++ b/storage/feed.go @@ -242,10 +242,11 @@ func (s *Storage) CreateFeed(feed *model.Feed) error { ignore_http_cache, allow_self_signed_certificates, fetch_via_proxy, - hide_globally + hide_globally, + url_rewrite_rules ) VALUES - ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21) + ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22) RETURNING id ` @@ -272,6 +273,7 @@ func (s *Storage) CreateFeed(feed *model.Feed) error { feed.AllowSelfSignedCertificates, feed.FetchViaProxy, feed.HideGlobally, + feed.UrlRewriteRules, ).Scan(&feed.ID) if err != nil { return fmt.Errorf(`store: unable to create feed %q: %v`, feed.FeedURL, err) @@ -330,9 +332,10 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) { ignore_http_cache=$21, allow_self_signed_certificates=$22, fetch_via_proxy=$23, - hide_globally=$24 + hide_globally=$24, + url_rewrite_rules=$25 WHERE - id=$25 AND user_id=$26 + id=$26 AND user_id=$27 ` _, err = s.db.Exec(query, feed.FeedURL, @@ -359,6 +362,7 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) { feed.AllowSelfSignedCertificates, feed.FetchViaProxy, feed.HideGlobally, + feed.UrlRewriteRules, feed.ID, feed.UserID, ) diff --git a/storage/feed_query_builder.go b/storage/feed_query_builder.go index 18c00049..4f3194db 100644 --- a/storage/feed_query_builder.go +++ b/storage/feed_query_builder.go @@ -157,6 +157,7 @@ func (f *FeedQueryBuilder) GetFeeds() (model.Feeds, error) { f.rewrite_rules, f.blocklist_rules, f.keeplist_rules, + f.url_rewrite_rules, f.crawler, f.user_agent, f.cookie, @@ -219,6 +220,7 @@ func (f *FeedQueryBuilder) GetFeeds() (model.Feeds, error) { &feed.RewriteRules, &feed.BlocklistRules, &feed.KeeplistRules, + &feed.UrlRewriteRules, &feed.Crawler, &feed.UserAgent, &feed.Cookie, diff --git a/template/templates/views/add_subscription.html b/template/templates/views/add_subscription.html index 7edff42c..4da9d327 100644 --- a/template/templates/views/add_subscription.html +++ b/template/templates/views/add_subscription.html @@ -97,6 +97,9 @@ + + + diff --git a/template/templates/views/choose_subscription.html b/template/templates/views/choose_subscription.html index 0a03b51d..c4463071 100644 --- a/template/templates/views/choose_subscription.html +++ b/template/templates/views/choose_subscription.html @@ -17,6 +17,7 @@ + {{ if .form.FetchViaProxy }} {{ end }} diff --git a/template/templates/views/edit_feed.html b/template/templates/views/edit_feed.html index f29e33e5..f191329b 100644 --- a/template/templates/views/edit_feed.html +++ b/template/templates/views/edit_feed.html @@ -32,7 +32,7 @@ {{ if .errorMessage }}
{{ t .errorMessage }}
{{ end }} - + + + + @@ -118,7 +121,7 @@ {{ end }} - + {{ if not .form.CategoryHidden }} {{ end }} diff --git a/ui/feed_edit.go b/ui/feed_edit.go index c2955b2b..ed9e5378 100644 --- a/ui/feed_edit.go +++ b/ui/feed_edit.go @@ -48,6 +48,7 @@ func (h *handler) showEditFeedPage(w http.ResponseWriter, r *http.Request) { RewriteRules: feed.RewriteRules, BlocklistRules: feed.BlocklistRules, KeeplistRules: feed.KeeplistRules, + UrlRewriteRules: feed.UrlRewriteRules, Crawler: feed.Crawler, UserAgent: feed.UserAgent, Cookie: feed.Cookie, diff --git a/ui/feed_update.go b/ui/feed_update.go index e54e58f9..183c30c6 100644 --- a/ui/feed_update.go +++ b/ui/feed_update.go @@ -58,12 +58,13 @@ func (h *handler) updateFeed(w http.ResponseWriter, r *http.Request) { view.Set("defaultUserAgent", config.Opts.HTTPClientUserAgent()) feedModificationRequest := &model.FeedModificationRequest{ - FeedURL: model.OptionalString(feedForm.FeedURL), - SiteURL: model.OptionalString(feedForm.SiteURL), - Title: model.OptionalString(feedForm.Title), - CategoryID: model.OptionalInt64(feedForm.CategoryID), - BlocklistRules: model.OptionalString(feedForm.BlocklistRules), - KeeplistRules: model.OptionalString(feedForm.KeeplistRules), + FeedURL: model.OptionalString(feedForm.FeedURL), + SiteURL: model.OptionalString(feedForm.SiteURL), + Title: model.OptionalString(feedForm.Title), + CategoryID: model.OptionalInt64(feedForm.CategoryID), + BlocklistRules: model.OptionalString(feedForm.BlocklistRules), + KeeplistRules: model.OptionalString(feedForm.KeeplistRules), + UrlRewriteRules: model.OptionalString(feedForm.UrlRewriteRules), } if validationErr := validator.ValidateFeedModification(h.store, loggedUser.ID, feedModificationRequest); validationErr != nil { diff --git a/ui/form/feed.go b/ui/form/feed.go index b89f1249..53181a7f 100644 --- a/ui/form/feed.go +++ b/ui/form/feed.go @@ -20,6 +20,7 @@ type FeedForm struct { RewriteRules string BlocklistRules string KeeplistRules string + UrlRewriteRules string Crawler bool UserAgent string Cookie string @@ -44,6 +45,7 @@ func (f FeedForm) Merge(feed *model.Feed) *model.Feed { feed.RewriteRules = f.RewriteRules feed.BlocklistRules = f.BlocklistRules feed.KeeplistRules = f.KeeplistRules + feed.UrlRewriteRules = f.UrlRewriteRules feed.Crawler = f.Crawler feed.UserAgent = f.UserAgent feed.Cookie = f.Cookie @@ -75,6 +77,7 @@ func NewFeedForm(r *http.Request) *FeedForm { RewriteRules: r.FormValue("rewrite_rules"), BlocklistRules: r.FormValue("blocklist_rules"), KeeplistRules: r.FormValue("keeplist_rules"), + UrlRewriteRules: r.FormValue("urlrewrite_rules"), Crawler: r.FormValue("crawler") == "1", CategoryID: int64(categoryID), Username: r.FormValue("feed_username"), diff --git a/ui/form/subscription.go b/ui/form/subscription.go index 769dfb2e..642c9534 100644 --- a/ui/form/subscription.go +++ b/ui/form/subscription.go @@ -27,6 +27,7 @@ type SubscriptionForm struct { RewriteRules string BlocklistRules string KeeplistRules string + UrlRewriteRules string } // Validate makes sure the form values are valid. @@ -47,6 +48,10 @@ func (s *SubscriptionForm) Validate() error { return errors.NewLocalizedError("error.feed_invalid_keeplist_rule") } + if !validator.IsValidRegex(s.UrlRewriteRules) { + return errors.NewLocalizedError("error.feed_invalid_urlrewrite_rule") + } + return nil } @@ -71,5 +76,6 @@ func NewSubscriptionForm(r *http.Request) *SubscriptionForm { RewriteRules: r.FormValue("rewrite_rules"), BlocklistRules: r.FormValue("blocklist_rules"), KeeplistRules: r.FormValue("keeplist_rules"), + UrlRewriteRules: r.FormValue("urlrewrite_rules"), } } diff --git a/ui/subscription_choose.go b/ui/subscription_choose.go index bc752efb..6d106330 100644 --- a/ui/subscription_choose.go +++ b/ui/subscription_choose.go @@ -62,6 +62,7 @@ func (h *handler) showChooseSubscriptionPage(w http.ResponseWriter, r *http.Requ RewriteRules: subscriptionForm.RewriteRules, BlocklistRules: subscriptionForm.BlocklistRules, KeeplistRules: subscriptionForm.KeeplistRules, + UrlRewriteRules: subscriptionForm.UrlRewriteRules, FetchViaProxy: subscriptionForm.FetchViaProxy, }) if err != nil { diff --git a/ui/subscription_submit.go b/ui/subscription_submit.go index e8ad2939..293b6c94 100644 --- a/ui/subscription_submit.go +++ b/ui/subscription_submit.go @@ -91,6 +91,7 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) { RewriteRules: subscriptionForm.RewriteRules, BlocklistRules: subscriptionForm.BlocklistRules, KeeplistRules: subscriptionForm.KeeplistRules, + UrlRewriteRules: subscriptionForm.UrlRewriteRules, FetchViaProxy: subscriptionForm.FetchViaProxy, }) if err != nil {