1
0
Fork 0
miniflux/reader/rss/rss.go

236 lines
5.2 KiB
Go
Raw Normal View History

2017-11-20 00:10:04 -05:00
// Copyright 2017 Frédéric Guillot. All rights reserved.
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package rss
import (
"encoding/xml"
"log"
"path"
"strconv"
"strings"
"time"
2017-11-20 18:15:10 -05:00
"github.com/miniflux/miniflux2/helper"
"github.com/miniflux/miniflux2/model"
"github.com/miniflux/miniflux2/reader/date"
2017-11-20 18:15:10 -05:00
"github.com/miniflux/miniflux2/reader/processor"
)
2017-11-20 00:10:04 -05:00
2017-11-20 18:15:10 -05:00
type rssFeed struct {
2017-11-20 00:10:04 -05:00
XMLName xml.Name `xml:"rss"`
Version string `xml:"version,attr"`
Title string `xml:"channel>title"`
2017-11-20 18:15:10 -05:00
Links []rssLink `xml:"channel>link"`
2017-11-20 00:10:04 -05:00
Language string `xml:"channel>language"`
Description string `xml:"channel>description"`
PubDate string `xml:"channel>pubDate"`
ItunesAuthor string `xml:"http://www.itunes.com/dtds/podcast-1.0.dtd channel>author"`
2017-11-20 18:15:10 -05:00
Items []rssItem `xml:"channel>item"`
2017-11-20 00:10:04 -05:00
}
2017-11-20 18:15:10 -05:00
type rssLink struct {
XMLName xml.Name
Data string `xml:",chardata"`
Href string `xml:"href,attr"`
2017-11-20 18:48:26 -05:00
Rel string `xml:"rel,attr"`
2017-11-20 18:15:10 -05:00
}
type rssItem struct {
GUID string `xml:"guid"`
2017-11-20 00:10:04 -05:00
Title string `xml:"title"`
2017-11-20 18:48:26 -05:00
Links []rssLink `xml:"link"`
2017-11-20 00:10:04 -05:00
OriginalLink string `xml:"http://rssnamespace.org/feedburner/ext/1.0 origLink"`
Description string `xml:"description"`
Content string `xml:"http://purl.org/rss/1.0/modules/content/ encoded"`
PubDate string `xml:"pubDate"`
Date string `xml:"http://purl.org/dc/elements/1.1/ date"`
2017-11-20 18:15:10 -05:00
Authors []rssAuthor `xml:"author"`
2017-11-20 00:10:04 -05:00
Creator string `xml:"http://purl.org/dc/elements/1.1/ creator"`
2017-11-20 18:15:10 -05:00
Enclosures []rssEnclosure `xml:"enclosure"`
2017-11-20 00:10:04 -05:00
OrigEnclosureLink string `xml:"http://rssnamespace.org/feedburner/ext/1.0 origEnclosureLink"`
}
2017-11-20 18:15:10 -05:00
type rssAuthor struct {
2017-11-20 00:10:04 -05:00
XMLName xml.Name
Data string `xml:",chardata"`
Name string `xml:"name"`
}
2017-11-20 18:15:10 -05:00
type rssEnclosure struct {
URL string `xml:"url,attr"`
2017-11-20 00:10:04 -05:00
Type string `xml:"type,attr"`
Length string `xml:"length,attr"`
}
2017-11-20 18:15:10 -05:00
func (r *rssFeed) GetSiteURL() string {
2017-11-20 18:48:26 -05:00
for _, element := range r.Links {
if element.XMLName.Space == "" {
2017-11-22 17:52:31 -05:00
return strings.TrimSpace(element.Data)
2017-11-20 00:10:04 -05:00
}
}
return ""
}
2017-11-20 18:15:10 -05:00
func (r *rssFeed) GetFeedURL() string {
2017-11-20 18:48:26 -05:00
for _, element := range r.Links {
if element.XMLName.Space == "http://www.w3.org/2005/Atom" {
2017-11-22 17:52:31 -05:00
return strings.TrimSpace(element.Href)
2017-11-20 00:10:04 -05:00
}
}
return ""
}
2017-11-20 18:15:10 -05:00
func (r *rssFeed) Transform() *model.Feed {
2017-11-20 00:10:04 -05:00
feed := new(model.Feed)
feed.SiteURL = r.GetSiteURL()
feed.FeedURL = r.GetFeedURL()
2017-11-22 17:52:31 -05:00
feed.Title = strings.TrimSpace(r.Title)
2017-11-20 00:10:04 -05:00
if feed.Title == "" {
feed.Title = feed.SiteURL
}
for _, item := range r.Items {
entry := item.Transform()
if entry.Author == "" && r.ItunesAuthor != "" {
entry.Author = r.ItunesAuthor
}
2017-11-22 17:52:31 -05:00
entry.Author = strings.TrimSpace(entry.Author)
2017-11-20 00:10:04 -05:00
2017-11-20 18:15:10 -05:00
if entry.URL == "" {
entry.URL = feed.SiteURL
}
2017-11-20 00:10:04 -05:00
feed.Entries = append(feed.Entries, entry)
}
return feed
}
2017-11-22 17:52:31 -05:00
2017-11-20 18:48:26 -05:00
func (r *rssItem) GetDate() time.Time {
value := r.PubDate
if r.Date != "" {
value = r.Date
2017-11-20 00:10:04 -05:00
}
if value != "" {
result, err := date.Parse(value)
if err != nil {
log.Println(err)
return time.Now()
}
return result
}
return time.Now()
}
2017-11-20 18:48:26 -05:00
func (r *rssItem) GetAuthor() string {
for _, element := range r.Authors {
2017-11-20 00:10:04 -05:00
if element.Name != "" {
return element.Name
}
if element.Data != "" {
return element.Data
}
}
2017-11-20 18:48:26 -05:00
return r.Creator
2017-11-20 00:10:04 -05:00
}
2017-11-20 18:48:26 -05:00
func (r *rssItem) GetHash() string {
for _, value := range []string{r.GUID, r.GetURL()} {
2017-11-20 00:10:04 -05:00
if value != "" {
return helper.Hash(value)
}
}
return ""
}
2017-11-20 18:48:26 -05:00
func (r *rssItem) GetContent() string {
if r.Content != "" {
return r.Content
2017-11-20 00:10:04 -05:00
}
2017-11-20 18:48:26 -05:00
return r.Description
2017-11-20 00:10:04 -05:00
}
2017-11-20 18:48:26 -05:00
func (r *rssItem) GetURL() string {
if r.OriginalLink != "" {
return r.OriginalLink
}
for _, link := range r.Links {
if link.XMLName.Space == "http://www.w3.org/2005/Atom" && link.Href != "" && isValidLinkRelation(link.Rel) {
2017-11-22 17:52:31 -05:00
return strings.TrimSpace(link.Href)
2017-11-20 18:48:26 -05:00
}
if link.Data != "" {
2017-11-22 17:52:31 -05:00
return strings.TrimSpace(link.Data)
2017-11-20 18:48:26 -05:00
}
2017-11-20 00:10:04 -05:00
}
2017-11-20 18:48:26 -05:00
return ""
2017-11-20 00:10:04 -05:00
}
2017-11-20 18:48:26 -05:00
func (r *rssItem) GetEnclosures() model.EnclosureList {
2017-11-20 00:10:04 -05:00
enclosures := make(model.EnclosureList, 0)
2017-11-20 18:48:26 -05:00
for _, enclosure := range r.Enclosures {
2017-11-20 00:10:04 -05:00
length, _ := strconv.Atoi(enclosure.Length)
2017-11-20 18:15:10 -05:00
enclosureURL := enclosure.URL
2017-11-20 00:10:04 -05:00
2017-11-20 18:48:26 -05:00
if r.OrigEnclosureLink != "" {
filename := path.Base(r.OrigEnclosureLink)
2017-11-20 00:10:04 -05:00
if strings.Contains(enclosureURL, filename) {
2017-11-20 18:48:26 -05:00
enclosureURL = r.OrigEnclosureLink
2017-11-20 00:10:04 -05:00
}
}
enclosures = append(enclosures, &model.Enclosure{
URL: enclosureURL,
MimeType: enclosure.Type,
Size: length,
})
}
return enclosures
}
2017-11-20 18:48:26 -05:00
func (r *rssItem) Transform() *model.Entry {
2017-11-20 00:10:04 -05:00
entry := new(model.Entry)
2017-11-20 18:48:26 -05:00
entry.URL = r.GetURL()
entry.Date = r.GetDate()
entry.Author = r.GetAuthor()
entry.Hash = r.GetHash()
entry.Content = processor.ItemContentProcessor(entry.URL, r.GetContent())
2017-11-22 17:52:31 -05:00
entry.Title = strings.TrimSpace(r.Title)
2017-11-20 18:48:26 -05:00
entry.Enclosures = r.GetEnclosures()
2017-11-20 00:10:04 -05:00
if entry.Title == "" {
entry.Title = entry.URL
}
return entry
}
2017-11-20 18:48:26 -05:00
func isValidLinkRelation(rel string) bool {
switch rel {
case "", "alternate", "enclosure", "related", "self", "via":
return true
default:
if strings.HasPrefix(rel, "http") {
return true
}
return false
}
}