forks/miniflux
1
0
Fork 0
Code Releases Activity

Add the possibility to override default user agent for each feed

Browse source
This commit is contained in:
Patrick 2018-09-20 03:19:24 +02:00 • committed by Frédéric Guillot
parent 1d335390c2
commit 2538eea177
29 changed files with 129 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

8
reader/processor/processor.go
View file

@ -21,6 +21,7 @@ type FeedProcessor struct {
scraperRules string
rewriteRules string
crawler bool
userAgent string
}
// WithCrawler enables the crawler.
@ -33,6 +34,11 @@ func (f *FeedProcessor) WithScraperRules(rules string) {
f.scraperRules = rules
}
// WithUserAgent sets the User-Agent header for fetching article content.
func (f *FeedProcessor) WithUserAgent(userAgent string) {
f.userAgent = userAgent
}
// WithRewriteRules adds rewrite rules to the processing.
func (f *FeedProcessor) WithRewriteRules(rules string) {
f.rewriteRules = rules
@ -45,7 +51,7 @@ func (f *FeedProcessor) Process() {
if f.store.EntryURLExists(f.userID, entry.URL) {
logger.Debug(`[FeedProcessor] Do not crawl existing entry URL: "%s"`, entry.URL)
} else {
content, err := scraper.Fetch(entry.URL, f.scraperRules)
content, err := scraper.Fetch(entry.URL, f.scraperRules, f.userAgent)
if err != nil {
logger.Error("[FeedProcessor] %v", err)
} else {