forks/miniflux
1
0
Fork 0
Code Releases Activity

Handle RSS feeds without entry links

Browse source
This commit is contained in:
Frédéric Guillot 2017-11-20 15:15:10 -08:00
parent a76c2a8c22
commit cf8af56a99
11 changed files with 83 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
locale/translations.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2017-11-20 14:18:33.283078515 -0800 PST m=+0.039164593 // 2017-11-20 15:11:09.541312146 -0800 PST m=+0.018445746
package locale package locale
@ -59,7 +59,7 @@ var Translations = map[string]string{
"%d days ago": "il y a %d jours", "%d days ago": "il y a %d jours",
"%d weeks ago": "il y a %d semaines", "%d weeks ago": "il y a %d semaines",
"%d months ago": "il y a %d mois", "%d months ago": "il y a %d mois",
"%d years ago": "il y a %d années", "%d years ago": "il y a %d ans",
"Date": "Date", "Date": "Date",
"IP Address": "Adresse IP", "IP Address": "Adresse IP",
"User Agent": "Navigateur Web", "User Agent": "Navigateur Web",
@ -133,5 +133,5 @@ var Translations = map[string]string{
var TranslationsChecksums = map[string]string{ var TranslationsChecksums = map[string]string{
"en_US": "6fe95384260941e8a5a3c695a655a932e0a8a6a572c1e45cb2b1ae8baa01b897", "en_US": "6fe95384260941e8a5a3c695a655a932e0a8a6a572c1e45cb2b1ae8baa01b897",
"fr_FR": "8b38f8e33f70c463560b49e7f408509304e4d8f61ae78ca26e33bbba28a9ec76", "fr_FR": "ec81374ce72a1cf3a06bda027faa338745a77035f8bd3504ad4d452958260525",
} }

2
locale/translations/fr_FR.json
View file

@ -43,7 +43,7 @@
"%d days ago": "il y a %d jours", "%d days ago": "il y a %d jours",
"%d weeks ago": "il y a %d semaines", "%d weeks ago": "il y a %d semaines",
"%d months ago": "il y a %d mois", "%d months ago": "il y a %d mois",
"%d years ago": "il y a %d années", "%d years ago": "il y a %d ans",
"Date": "Date", "Date": "Date",
"IP Address": "Adresse IP", "IP Address": "Adresse IP",
"User Agent": "Navigateur Web", "User Agent": "Navigateur Web",

11
reader/feed/rss/parser.go
View file

@ -7,22 +7,23 @@ package rss
import ( import (
"encoding/xml" "encoding/xml"
"fmt" "fmt"
"github.com/miniflux/miniflux2/model"
"io" "io"
"github.com/miniflux/miniflux2/model"
"golang.org/x/net/html/charset" "golang.org/x/net/html/charset"
) )
// Parse returns a normalized feed struct. // Parse returns a normalized feed struct.
func Parse(data io.Reader) (*model.Feed, error) { func Parse(data io.Reader) (*model.Feed, error) {
rssFeed := new(RssFeed) feed := new(rssFeed)
decoder := xml.NewDecoder(data) decoder := xml.NewDecoder(data)
decoder.CharsetReader = charset.NewReaderLabel decoder.CharsetReader = charset.NewReaderLabel
err := decoder.Decode(rssFeed) err := decoder.Decode(feed)
if err != nil { if err != nil {
return nil, fmt.Errorf("Unable to parse RSS feed: %v", err) return nil, fmt.Errorf("unable to parse RSS feed: %v", err)
} }
return rssFeed.Transform(), nil return feed.Transform(), nil
} }

25
reader/feed/rss/parser_test.go
View file

@ -136,6 +136,31 @@ func TestParseEntryWithoutTitle(t *testing.T) {
} }
} }
func TestParseEntryWithoutLink(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0">
<channel>
<link>https://example.org/</link>
<item>
<guid isPermaLink="false">1234</guid>
</item>
</channel>
</rss>`
feed, err := Parse(bytes.NewBufferString(data))
if err != nil {
t.Error(err)
}
if feed.Entries[0].URL != "https://example.org/" {
t.Errorf("Incorrect entry link, got: %s", feed.Entries[0].URL)
}
if feed.Entries[0].Hash != "03ac674216f3e15c761ee1a5e255f067953623c8b388b4459e13f978d7c846f4" {
t.Errorf("Incorrect entry hash, got: %s", feed.Entries[0].Hash)
}
}
func TestParseFeedURLWithAtomLink(t *testing.T) { func TestParseFeedURLWithAtomLink(t *testing.T) {
data := `<?xml version="1.0" encoding="utf-8"?> data := `<?xml version="1.0" encoding="utf-8"?>
<rss xmlns:atom="http://www.w3.org/2005/Atom" version="2.0"> <rss xmlns:atom="http://www.w3.org/2005/Atom" version="2.0">

79
reader/feed/rss/rss.go
View file

@ -6,38 +6,39 @@ package rss
import ( import (
"encoding/xml" "encoding/xml"
"github.com/miniflux/miniflux2/helper"
"github.com/miniflux/miniflux2/model"
"github.com/miniflux/miniflux2/reader/feed/date"
"github.com/miniflux/miniflux2/reader/processor"
"github.com/miniflux/miniflux2/reader/sanitizer"
"log" "log"
"path" "path"
"strconv" "strconv"
"strings" "strings"
"time" "time"
"github.com/miniflux/miniflux2/helper"
"github.com/miniflux/miniflux2/model"
"github.com/miniflux/miniflux2/reader/feed/date"
"github.com/miniflux/miniflux2/reader/processor"
"github.com/miniflux/miniflux2/reader/sanitizer"
) )
type RssLink struct { type rssFeed struct {
XMLName xml.Name `xml:"rss"`
Version string `xml:"version,attr"`
Title string `xml:"channel>title"`
Links []rssLink `xml:"channel>link"`
Language string `xml:"channel>language"`
Description string `xml:"channel>description"`
PubDate string `xml:"channel>pubDate"`
ItunesAuthor string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd channel>author"`
Items []rssItem `xml:"channel>item"`
}
type rssLink struct {
XMLName xml.Name XMLName xml.Name
Data string `xml:",chardata"` Data string `xml:",chardata"`
Href string `xml:"href,attr"` Href string `xml:"href,attr"`
} }
type RssFeed struct { type rssItem struct {
XMLName xml.Name `xml:"rss"` GUID string `xml:"guid"`
Version string `xml:"version,attr"`
Title string `xml:"channel>title"`
Links []RssLink `xml:"channel>link"`
Language string `xml:"channel>language"`
Description string `xml:"channel>description"`
PubDate string `xml:"channel>pubDate"`
ItunesAuthor string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd channel>author"`
Items []RssItem `xml:"channel>item"`
}
type RssItem struct {
Guid string `xml:"guid"`
Title string `xml:"title"` Title string `xml:"title"`
Link string `xml:"link"` Link string `xml:"link"`
OriginalLink string `xml:"http://rssnamespace.org/feedburner/ext/1.0 origLink"` OriginalLink string `xml:"http://rssnamespace.org/feedburner/ext/1.0 origLink"`
@ -45,25 +46,25 @@ type RssItem struct {
Content string `xml:"http://purl.org/rss/1.0/modules/content/ encoded"` Content string `xml:"http://purl.org/rss/1.0/modules/content/ encoded"`
PubDate string `xml:"pubDate"` PubDate string `xml:"pubDate"`
Date string `xml:"http://purl.org/dc/elements/1.1/ date"` Date string `xml:"http://purl.org/dc/elements/1.1/ date"`
Authors []RssAuthor `xml:"author"` Authors []rssAuthor `xml:"author"`
Creator string `xml:"http://purl.org/dc/elements/1.1/ creator"` Creator string `xml:"http://purl.org/dc/elements/1.1/ creator"`
Enclosures []RssEnclosure `xml:"enclosure"` Enclosures []rssEnclosure `xml:"enclosure"`
OrigEnclosureLink string `xml:"http://rssnamespace.org/feedburner/ext/1.0 origEnclosureLink"` OrigEnclosureLink string `xml:"http://rssnamespace.org/feedburner/ext/1.0 origEnclosureLink"`
} }
type RssAuthor struct { type rssAuthor struct {
XMLName xml.Name XMLName xml.Name
Data string `xml:",chardata"` Data string `xml:",chardata"`
Name string `xml:"name"` Name string `xml:"name"`
} }
type RssEnclosure struct { type rssEnclosure struct {
Url string `xml:"url,attr"` URL string `xml:"url,attr"`
Type string `xml:"type,attr"` Type string `xml:"type,attr"`
Length string `xml:"length,attr"` Length string `xml:"length,attr"`
} }
func (r *RssFeed) GetSiteURL() string { func (r *rssFeed) GetSiteURL() string {
for _, elem := range r.Links { for _, elem := range r.Links {
if elem.XMLName.Space == "" { if elem.XMLName.Space == "" {
return elem.Data return elem.Data
@ -73,7 +74,7 @@ func (r *RssFeed) GetSiteURL() string {
return "" return ""
} }
func (r *RssFeed) GetFeedURL() string { func (r *rssFeed) GetFeedURL() string {
for _, elem := range r.Links { for _, elem := range r.Links {
if elem.XMLName.Space == "http://www.w3.org/2005/Atom" { if elem.XMLName.Space == "http://www.w3.org/2005/Atom" {
return elem.Href return elem.Href
@ -83,7 +84,7 @@ func (r *RssFeed) GetFeedURL() string {
return "" return ""
} }
func (r *RssFeed) Transform() *model.Feed { func (r *rssFeed) Transform() *model.Feed {
feed := new(model.Feed) feed := new(model.Feed)
feed.SiteURL = r.GetSiteURL() feed.SiteURL = r.GetSiteURL()
feed.FeedURL = r.GetFeedURL() feed.FeedURL = r.GetFeedURL()
@ -101,12 +102,16 @@ func (r *RssFeed) Transform() *model.Feed {
} }
entry.Author = sanitizer.StripTags(entry.Author) entry.Author = sanitizer.StripTags(entry.Author)
if entry.URL == "" {
entry.URL = feed.SiteURL
}
feed.Entries = append(feed.Entries, entry) feed.Entries = append(feed.Entries, entry)
} }
return feed return feed
} }
func (i *RssItem) GetDate() time.Time { func (i *rssItem) GetDate() time.Time {
value := i.PubDate value := i.PubDate
if i.Date != "" { if i.Date != "" {
value = i.Date value = i.Date
@ -125,7 +130,7 @@ func (i *RssItem) GetDate() time.Time {
return time.Now() return time.Now()
} }
func (i *RssItem) GetAuthor() string { func (i *rssItem) GetAuthor() string {
for _, element := range i.Authors { for _, element := range i.Authors {
if element.Name != "" { if element.Name != "" {
return element.Name return element.Name
@ -139,8 +144,8 @@ func (i *RssItem) GetAuthor() string {
return i.Creator return i.Creator
} }
func (i *RssItem) GetHash() string { func (i *rssItem) GetHash() string {
for _, value := range []string{i.Guid, i.Link} { for _, value := range []string{i.GUID, i.Link} {
if value != "" { if value != "" {
return helper.Hash(value) return helper.Hash(value)
} }
@ -149,7 +154,7 @@ func (i *RssItem) GetHash() string {
return "" return ""
} }
func (i *RssItem) GetContent() string { func (i *rssItem) GetContent() string {
if i.Content != "" { if i.Content != "" {
return i.Content return i.Content
} }
@ -157,7 +162,7 @@ func (i *RssItem) GetContent() string {
return i.Description return i.Description
} }
func (i *RssItem) GetURL() string { func (i *rssItem) GetURL() string {
if i.OriginalLink != "" { if i.OriginalLink != "" {
return i.OriginalLink return i.OriginalLink
} }
@ -165,12 +170,12 @@ func (i *RssItem) GetURL() string {
return i.Link return i.Link
} }
func (i *RssItem) GetEnclosures() model.EnclosureList { func (i *rssItem) GetEnclosures() model.EnclosureList {
enclosures := make(model.EnclosureList, 0) enclosures := make(model.EnclosureList, 0)
for _, enclosure := range i.Enclosures { for _, enclosure := range i.Enclosures {
length, _ := strconv.Atoi(enclosure.Length) length, _ := strconv.Atoi(enclosure.Length)
enclosureURL := enclosure.Url enclosureURL := enclosure.URL
if i.OrigEnclosureLink != "" { if i.OrigEnclosureLink != "" {
filename := path.Base(i.OrigEnclosureLink) filename := path.Base(i.OrigEnclosureLink)
@ -189,7 +194,7 @@ func (i *RssItem) GetEnclosures() model.EnclosureList {
return enclosures return enclosures
} }
func (i *RssItem) Transform() *model.Entry { func (i *rssItem) Transform() *model.Entry {
entry := new(model.Entry) entry := new(model.Entry)
entry.URL = i.GetURL() entry.URL = i.GetURL()
entry.Date = i.GetDate() entry.Date = i.GetDate()

2
server/static/bin.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2017-11-20 14:18:33.249018092 -0800 PST m=+0.005104170 // 2017-11-20 15:11:09.5282708 -0800 PST m=+0.005404400
package static package static

2
server/static/css.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2017-11-20 14:18:33.25118969 -0800 PST m=+0.007275768 // 2017-11-20 15:11:09.529138622 -0800 PST m=+0.006272222
package static package static

2
server/static/js.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2017-11-20 14:18:33.255571671 -0800 PST m=+0.011657749 // 2017-11-20 15:11:09.530927342 -0800 PST m=+0.008060942
package static package static

2
server/template/common.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2017-11-20 14:18:33.28176883 -0800 PST m=+0.037854908 // 2017-11-20 15:11:09.54007082 -0800 PST m=+0.017204420
package template package template

2
server/template/views.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2017-11-20 14:18:33.257809595 -0800 PST m=+0.013895673 // 2017-11-20 15:11:09.531826622 -0800 PST m=+0.008960222
package template package template

2
sql/sql.go
View file

@ -1,5 +1,5 @@
// Code generated by go generate; DO NOT EDIT. // Code generated by go generate; DO NOT EDIT.
// 2017-11-20 14:18:33.247054842 -0800 PST m=+0.003140920 // 2017-11-20 15:11:09.527012847 -0800 PST m=+0.004146447
package sql package sql