forks/miniflux
1
0
Fork 0
Code Releases Activity

Add the possibility to add rules during feed creation

Browse source
This commit is contained in:
Frédéric Guillot 2019-11-29 11:17:14 -08:00
parent 8028cc764f
commit 69aa650203
13 changed files with 74 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
api/feed.go
View file

@ -49,6 +49,8 @@ func (h *handler) createFeed(w http.ResponseWriter, r *http.Request) {
feedInfo.UserAgent,
feedInfo.Username,
feedInfo.Password,
feedInfo.ScraperRules,
feedInfo.RewriteRules,
)
if err != nil {
json.ServerError(w, r, err)

14
api/payload.go
View file

@ -24,12 +24,14 @@ type entriesResponse struct {
}
type feedCreation struct {
FeedURL string `json:"feed_url"`
CategoryID int64 `json:"category_id"`
UserAgent string `json:"user_agent"`
Username string `json:"username"`
Password string `json:"password"`
Crawler bool `json:"crawler"`
FeedURL string `json:"feed_url"`
CategoryID int64 `json:"category_id"`
UserAgent string `json:"user_agent"`
Username string `json:"username"`
Password string `json:"password"`
Crawler bool `json:"crawler"`
ScraperRules string `json:"scraper_rules"`
RewriteRules string `json:"rewrite_rules"`
}
type subscriptionDiscovery struct {

4
model/feed.go
View file

@ -61,11 +61,13 @@ func (f *Feed) WithCategoryID(categoryID int64) {
}
// WithBrowsingParameters defines browsing parameters.
func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, password string) {
func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, password, scraperRules, rewriteRules string) {
f.Crawler = crawler
f.UserAgent = userAgent
f.Username = username
f.Password = password
f.ScraperRules = scraperRules
f.RewriteRules = rewriteRules
}
// WithError adds a new error message and increment the error counter.

10
model/feed_test.go
View file

@ -44,7 +44,7 @@ func TestFeedCategorySetter(t *testing.T) {
func TestFeedBrowsingParams(t *testing.T) {
feed := &Feed{}
feed.WithBrowsingParameters(true, "Custom User Agent", "Username", "Secret")
feed.WithBrowsingParameters(true, "Custom User Agent", "Username", "Secret", "Some Rule", "Another Rule")
if !feed.Crawler {
t.Error(`The crawler must be activated`)
@ -61,6 +61,14 @@ func TestFeedBrowsingParams(t *testing.T) {
if feed.Password != "Secret" {
t.Error(`The password must be set`)
}
if feed.ScraperRules != "Some Rule" {
t.Errorf(`The scraper rules must be set`)
}
if feed.RewriteRules != "Another Rule" {
t.Errorf(`The rewrite rules must be set`)
}
}
func TestFeedErrorCounter(t *testing.T) {

8
reader/feed/handler.go
View file

@ -29,11 +29,11 @@ var (
// Handler contains all the logic to create and refresh feeds.
type Handler struct {
store *storage.Storage
store *storage.Storage
}
// CreateFeed fetch, parse and store a new feed.
func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password string) (*model.Feed, error) {
func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password, scraperRules, rewriteRules string) (*model.Feed, error) {
defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Handler:CreateFeed] feedUrl=%s", url))
if !h.store.CategoryExists(userID, categoryID) {
@ -59,7 +59,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool,
subscription.UserID = userID
subscription.WithCategoryID(categoryID)
subscription.WithBrowsingParameters(crawler, userAgent, username, password)
subscription.WithBrowsingParameters(crawler, userAgent, username, password, scraperRules, rewriteRules)
subscription.WithClientResponse(response)
subscription.CheckedNow()
@ -160,4 +160,4 @@ func checkFeedIcon(store *storage.Storage, feedID int64, websiteURL string) {
}
}
}
}
}

2
reader/rewrite/rewriter.go
View file

@ -37,7 +37,7 @@ func Rewriter(entryURL, entryContent, customRewriteRules string) string {
entryContent = addPDFLink(entryURL, entryContent)
case "nl2br":
entryContent = replaceLineFeeds(entryContent)
case "convert_text_link":
case "convert_text_link", "convert_text_links":
entryContent = replaceTextLinks(entryContent)
}
}

8
storage/feed.go
View file

@ -346,10 +346,12 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
user_agent,
username,
password,
disabled
disabled,
scraper_rules,
rewrite_rules
)
VALUES
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
RETURNING
id
`
@ -367,6 +369,8 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
feed.Username,
feed.Password,
feed.Disabled,
feed.ScraperRules,
feed.RewriteRules,
).Scan(&feed.ID)
if err != nil {
return fmt.Errorf(`store: unable to create feed %q: %v`, feed.FeedURL, err)

6
template/html/add_subscription.html
View file

@ -56,6 +56,12 @@
- Using a different input name doesn't change anything
-->
<input type="text" name="feed_password" id="form-feed-password" value="{{ .form.Password }}">
<label for="form-scraper-rules">{{ t "form.feed.label.scraper_rules" }}</label>
<input type="text" name="scraper_rules" id="form-scraper-rules" value="{{ .form.ScraperRules }}">
<label for="form-rewrite-rules">{{ t "form.feed.label.rewrite_rules" }}</label>
<input type="text" name="rewrite_rules" id="form-rewrite-rules" value="{{ .form.RewriteRules }}">
</div>
</details>

2
template/html/choose_subscription.html
View file

@ -22,6 +22,8 @@
<input type="hidden" name="user_agent" value="{{ .form.UserAgent }}">
<input type="hidden" name="feed_username" value="{{ .form.Username }}">
<input type="hidden" name="feed_password" value="{{ .form.Password }}">
<input type="hidden" name="scraper_rules" value="{{ .form.ScraperRules }}">
<input type="hidden" name="rewrite_rules" value="{{ .form.RewriteRules }}">
{{ if .form.Crawler }}
<input type="hidden" name="crawler" value="1">
{{ end }}

12
template/views.go
View file

@ -87,6 +87,12 @@ var templateViewsMap = map[string]string{
- Using a different input name doesn't change anything
-->
<input type="text" name="feed_password" id="form-feed-password" value="{{ .form.Password }}">
<label for="form-scraper-rules">{{ t "form.feed.label.scraper_rules" }}</label>
<input type="text" name="scraper_rules" id="form-scraper-rules" value="{{ .form.ScraperRules }}">
<label for="form-rewrite-rules">{{ t "form.feed.label.rewrite_rules" }}</label>
<input type="text" name="rewrite_rules" id="form-rewrite-rules" value="{{ .form.RewriteRules }}">
</div>
</details>
@ -311,6 +317,8 @@ var templateViewsMap = map[string]string{
<input type="hidden" name="user_agent" value="{{ .form.UserAgent }}">
<input type="hidden" name="feed_username" value="{{ .form.Username }}">
<input type="hidden" name="feed_password" value="{{ .form.Password }}">
<input type="hidden" name="scraper_rules" value="{{ .form.ScraperRules }}">
<input type="hidden" name="rewrite_rules" value="{{ .form.RewriteRules }}">
{{ if .form.Crawler }}
<input type="hidden" name="crawler" value="1">
{{ end }}
@ -1343,12 +1351,12 @@ var templateViewsMap = map[string]string{
var templateViewsMapChecksums = map[string]string{
"about": "4035658497363d7af7f79be83190404eb21ec633fe8ec636bdfc219d9fc78cfc",
"add_subscription": "a0f1d2bc02b6adc83dbeae593f74d9b936102cd6dd73302cdbec2137cafdcdd9",
"add_subscription": "5bae8c60989593257f515f9c73b35c854a94bee4b2fb08ad38aa93b17be9c1b5",
"bookmark_entries": "65588da78665699dd3f287f68325e9777d511f1a57fee4131a5bb6d00bb68df8",
"categories": "2c5dd0ed6355bd5acc393bbf6117d20458b5581aab82036008324f6bbbe2af75",
"category_entries": "dee7b9cd60c6c46f01dd4289940679df31c1fce28ce4aa7249fa459023e1eeb4",
"category_feeds": "527c2ffbc4fcec775071424ba1022ae003525dba53a28cc41f48fb7b30aa984b",
"choose_subscription": "33c04843d7c1b608d034e605e52681822fc6d79bc6b900c04915dd9ebae584e2",
"choose_subscription": "5f21556e6cecfd64b1cff30e22ef313d2f09698cebb03f711cbac8ec7f2c1d04",
"create_category": "6b22b5ce51abf4e225e23a79f81be09a7fb90acb265e93a8faf9446dff74018d",
"create_user": "9b73a55233615e461d1f07d99ad1d4d3b54532588ab960097ba3e090c85aaf3a",
"edit_category": "b1c0b38f1b714c5d884edcd61e5b5295a5f1c8b71c469b35391e4dcc97cc6d36",

28
ui/form/subscription.go
View file

@ -13,12 +13,14 @@ import (
// SubscriptionForm represents the subscription form.
type SubscriptionForm struct {
URL string
CategoryID int64
Crawler bool
UserAgent string
Username string
Password string
URL string
CategoryID int64
Crawler bool
UserAgent string
Username string
Password string
ScraperRules string
RewriteRules string
}
// Validate makes sure the form values are valid.
@ -38,11 +40,13 @@ func NewSubscriptionForm(r *http.Request) *SubscriptionForm {
}
return &SubscriptionForm{
URL: r.FormValue("url"),
Crawler: r.FormValue("crawler") == "1",
CategoryID: int64(categoryID),
UserAgent: r.FormValue("user_agent"),
Username: r.FormValue("feed_username"),
Password: r.FormValue("feed_password"),
URL: r.FormValue("url"),
Crawler: r.FormValue("crawler") == "1",
CategoryID: int64(categoryID),
UserAgent: r.FormValue("user_agent"),
Username: r.FormValue("feed_username"),
Password: r.FormValue("feed_password"),
ScraperRules: r.FormValue("scraper_rules"),
RewriteRules: r.FormValue("rewrite_rules"),
}
}

4
ui/subscription_choose.go
View file

@ -8,8 +8,8 @@ import (
"net/http"
"miniflux.app/http/client"
"miniflux.app/http/response/html"
"miniflux.app/http/request"
"miniflux.app/http/response/html"
"miniflux.app/http/route"
"miniflux.app/ui/form"
"miniflux.app/ui/session"
@ -55,6 +55,8 @@ func (h *handler) showChooseSubscriptionPage(w http.ResponseWriter, r *http.Requ
subscriptionForm.UserAgent,
subscriptionForm.Username,
subscriptionForm.Password,
subscriptionForm.ScraperRules,
subscriptionForm.RewriteRules,
)
if err != nil {
view.Set("form", subscriptionForm)

6
ui/subscription_submit.go
View file

@ -2,14 +2,14 @@
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package ui // import "miniflux.app/ui"
package ui // import "miniflux.app/ui"
import (
"net/http"
"miniflux.app/http/client"
"miniflux.app/http/response/html"
"miniflux.app/http/request"
"miniflux.app/http/response/html"
"miniflux.app/http/route"
"miniflux.app/logger"
"miniflux.app/reader/subscription"
@ -80,6 +80,8 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) {
subscriptionForm.UserAgent,
subscriptionForm.Username,
subscriptionForm.Password,
subscriptionForm.ScraperRules,
subscriptionForm.RewriteRules,
)
if err != nil {
v.Set("form", subscriptionForm)