Add readability package to fetch original content
This commit is contained in:
Frédéric Guillot
parent
b75a9987ba
commit
7a35c58f53
17 changed files with 545 additions and 70 deletions
|
|
@ -1,5 +1,5 @@
|
|||
// Code generated by go generate; DO NOT EDIT.
|
||||
// 2017-12-04 21:22:23.44799753 -0800 PST m=+0.006646042
|
||||
// 2017-12-10 18:56:24.387844114 -0800 PST m=+0.029823201
|
||||
|
||||
package locale
|
||||
|
||||
|
|
@ -166,12 +166,13 @@ var translations = map[string]string{
|
|||
"Instapaper Password": "Mot de passe Instapaper",
|
||||
"Activate Fever API": "Activer l'API de Fever",
|
||||
"Fever Username": "Nom d'utilisateur pour l'API de Fever",
|
||||
"Fever Password": "Mot de passe pour l'API de Fever"
|
||||
"Fever Password": "Mot de passe pour l'API de Fever",
|
||||
"Fetch original content": "Récupérer le contenu original"
|
||||
}
|
||||
`,
|
||||
}
|
||||
|
||||
var translationsChecksums = map[string]string{
|
||||
"en_US": "6fe95384260941e8a5a3c695a655a932e0a8a6a572c1e45cb2b1ae8baa01b897",
|
||||
"fr_FR": "ef3d095f3e78d88a2746240769fa30d2e83c6519187d98e2193c9231dda5d882",
|
||||
"fr_FR": "fd629b171aefa50dd0a6100acaac8fbecbdf1a1d53e3fce984234565ec5bb5d5",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -150,5 +150,6 @@
|
|||
"Instapaper Password": "Mot de passe Instapaper",
|
||||
"Activate Fever API": "Activer l'API de Fever",
|
||||
"Fever Username": "Nom d'utilisateur pour l'API de Fever",
|
||||
"Fever Password": "Mot de passe pour l'API de Fever"
|
||||
"Fever Password": "Mot de passe pour l'API de Fever",
|
||||
"Fetch original content": "Récupérer le contenu original"
|
||||
}
|
||||
|
|
|
|||
306
reader/readability/readability.go
Normal file
306
reader/readability/readability.go
Normal file
|
|
@ -0,0 +1,306 @@
|
|||
// Copyright 2017 Frédéric Guillot. All rights reserved.
|
||||
// Use of this source code is governed by the Apache 2.0
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package readability
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultTagsToScore = "section,h2,h3,h4,h5,h6,p,td,pre,div"
|
||||
)
|
||||
|
||||
var (
|
||||
divToPElementsRegexp = regexp.MustCompile(`(?i)<(a|blockquote|dl|div|img|ol|p|pre|table|ul)`)
|
||||
sentenceRegexp = regexp.MustCompile(`\.( |$)`)
|
||||
|
||||
blacklistCandidatesRegexp = regexp.MustCompile(`(?i)popupbody|-ad|g-plus`)
|
||||
okMaybeItsACandidateRegexp = regexp.MustCompile(`(?i)and|article|body|column|main|shadow`)
|
||||
unlikelyCandidatesRegexp = regexp.MustCompile(`(?i)banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|foot|header|legends|menu|modal|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote`)
|
||||
|
||||
negativeRegexp = regexp.MustCompile(`(?i)hidden|^hid$|hid$|hid|^hid |banner|combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|modal|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget|byline|author|dateline|writtenby|p-author`)
|
||||
positiveRegexp = regexp.MustCompile(`(?i)article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story`)
|
||||
)
|
||||
|
||||
type candidate struct {
|
||||
selection *goquery.Selection
|
||||
score float32
|
||||
}
|
||||
|
||||
func (c *candidate) Node() *html.Node {
|
||||
return c.selection.Get(0)
|
||||
}
|
||||
|
||||
func (c *candidate) String() string {
|
||||
id, _ := c.selection.Attr("id")
|
||||
class, _ := c.selection.Attr("class")
|
||||
|
||||
if id != "" && class != "" {
|
||||
return fmt.Sprintf("%s#%s.%s => %f", c.Node().DataAtom, id, class, c.score)
|
||||
} else if id != "" {
|
||||
return fmt.Sprintf("%s#%s => %f", c.Node().DataAtom, id, c.score)
|
||||
} else if class != "" {
|
||||
return fmt.Sprintf("%s.%s => %f", c.Node().DataAtom, class, c.score)
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s => %f", c.Node().DataAtom, c.score)
|
||||
}
|
||||
|
||||
type candidateList map[*html.Node]*candidate
|
||||
|
||||
func (c candidateList) String() string {
|
||||
var output []string
|
||||
for _, candidate := range c {
|
||||
output = append(output, candidate.String())
|
||||
}
|
||||
|
||||
return strings.Join(output, ", ")
|
||||
}
|
||||
|
||||
// ExtractContent returns relevant content.
|
||||
func ExtractContent(page io.Reader) (string, error) {
|
||||
document, err := goquery.NewDocumentFromReader(page)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
document.Find("script,style,noscript").Each(func(i int, s *goquery.Selection) {
|
||||
removeNodes(s)
|
||||
})
|
||||
|
||||
transformMisusedDivsIntoParagraphs(document)
|
||||
removeUnlikelyCandidates(document)
|
||||
|
||||
candidates := getCandidates(document)
|
||||
log.Println("Candidates:", candidates)
|
||||
|
||||
topCandidate := getTopCandidate(document, candidates)
|
||||
log.Println("TopCandidate:", topCandidate)
|
||||
|
||||
output := getArticle(topCandidate, candidates)
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// Now that we have the top candidate, look through its siblings for content that might also be related.
|
||||
// Things like preambles, content split by ads that we removed, etc.
|
||||
func getArticle(topCandidate *candidate, candidates candidateList) string {
|
||||
output := bytes.NewBufferString("<div>")
|
||||
siblingScoreThreshold := float32(math.Max(10, float64(topCandidate.score*.2)))
|
||||
|
||||
topCandidate.selection.Siblings().Union(topCandidate.selection).Each(func(i int, s *goquery.Selection) {
|
||||
append := false
|
||||
node := s.Get(0)
|
||||
|
||||
if node == topCandidate.Node() {
|
||||
append = true
|
||||
} else if c, ok := candidates[node]; ok && c.score >= siblingScoreThreshold {
|
||||
append = true
|
||||
}
|
||||
|
||||
if s.Is("p") {
|
||||
linkDensity := getLinkDensity(s)
|
||||
content := s.Text()
|
||||
contentLength := len(content)
|
||||
|
||||
if contentLength >= 80 && linkDensity < .25 {
|
||||
append = true
|
||||
} else if contentLength < 80 && linkDensity == 0 && sentenceRegexp.MatchString(content) {
|
||||
append = true
|
||||
}
|
||||
}
|
||||
|
||||
if append {
|
||||
tag := "div"
|
||||
if s.Is("p") {
|
||||
tag = node.Data
|
||||
}
|
||||
|
||||
html, _ := s.Html()
|
||||
fmt.Fprintf(output, "<%s>%s</%s>", tag, html, tag)
|
||||
}
|
||||
})
|
||||
|
||||
output.Write([]byte("</div>"))
|
||||
return output.String()
|
||||
}
|
||||
|
||||
func removeUnlikelyCandidates(document *goquery.Document) {
|
||||
document.Find("*").Not("html,body").Each(func(i int, s *goquery.Selection) {
|
||||
class, _ := s.Attr("class")
|
||||
id, _ := s.Attr("id")
|
||||
str := class + id
|
||||
|
||||
if blacklistCandidatesRegexp.MatchString(str) || (unlikelyCandidatesRegexp.MatchString(str) && !okMaybeItsACandidateRegexp.MatchString(str)) {
|
||||
// log.Printf("Removing unlikely candidate - %s\n", str)
|
||||
removeNodes(s)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func getTopCandidate(document *goquery.Document, candidates candidateList) *candidate {
|
||||
var best *candidate
|
||||
|
||||
for _, c := range candidates {
|
||||
if best == nil {
|
||||
best = c
|
||||
} else if best.score < c.score {
|
||||
best = c
|
||||
}
|
||||
}
|
||||
|
||||
if best == nil {
|
||||
best = &candidate{document.Find("body"), 0}
|
||||
}
|
||||
|
||||
return best
|
||||
}
|
||||
|
||||
// Loop through all paragraphs, and assign a score to them based on how content-y they look.
|
||||
// Then add their score to their parent node.
|
||||
// A score is determined by things like number of commas, class names, etc.
|
||||
// Maybe eventually link density.
|
||||
func getCandidates(document *goquery.Document) candidateList {
|
||||
candidates := make(candidateList)
|
||||
|
||||
document.Find(defaultTagsToScore).Each(func(i int, s *goquery.Selection) {
|
||||
text := s.Text()
|
||||
|
||||
// If this paragraph is less than 25 characters, don't even count it.
|
||||
if len(text) < 25 {
|
||||
return
|
||||
}
|
||||
|
||||
parent := s.Parent()
|
||||
parentNode := parent.Get(0)
|
||||
|
||||
grandParent := parent.Parent()
|
||||
var grandParentNode *html.Node
|
||||
if grandParent.Length() > 0 {
|
||||
grandParentNode = grandParent.Get(0)
|
||||
}
|
||||
|
||||
if _, found := candidates[parentNode]; !found {
|
||||
candidates[parentNode] = scoreNode(parent)
|
||||
}
|
||||
|
||||
if grandParentNode != nil {
|
||||
if _, found := candidates[grandParentNode]; !found {
|
||||
candidates[grandParentNode] = scoreNode(grandParent)
|
||||
}
|
||||
}
|
||||
|
||||
// Add a point for the paragraph itself as a base.
|
||||
contentScore := float32(1.0)
|
||||
|
||||
// Add points for any commas within this paragraph.
|
||||
contentScore += float32(strings.Count(text, ",") + 1)
|
||||
|
||||
// For every 100 characters in this paragraph, add another point. Up to 3 points.
|
||||
contentScore += float32(math.Min(float64(int(len(text)/100.0)), 3))
|
||||
|
||||
candidates[parentNode].score += contentScore
|
||||
if grandParentNode != nil {
|
||||
candidates[grandParentNode].score += contentScore / 2.0
|
||||
}
|
||||
})
|
||||
|
||||
// Scale the final candidates score based on link density. Good content
|
||||
// should have a relatively small link density (5% or less) and be mostly
|
||||
// unaffected by this operation
|
||||
for _, candidate := range candidates {
|
||||
candidate.score = candidate.score * (1 - getLinkDensity(candidate.selection))
|
||||
}
|
||||
|
||||
return candidates
|
||||
}
|
||||
|
||||
func scoreNode(s *goquery.Selection) *candidate {
|
||||
c := &candidate{selection: s, score: 0}
|
||||
|
||||
switch s.Get(0).DataAtom.String() {
|
||||
case "div":
|
||||
c.score += 5
|
||||
case "pre", "td", "blockquote", "img":
|
||||
c.score += 3
|
||||
case "address", "ol", "ul", "dl", "dd", "dt", "li", "form":
|
||||
c.score -= 3
|
||||
case "h1", "h2", "h3", "h4", "h5", "h6", "th":
|
||||
c.score -= 5
|
||||
}
|
||||
|
||||
c.score += getClassWeight(s)
|
||||
return c
|
||||
}
|
||||
|
||||
// Get the density of links as a percentage of the content
|
||||
// This is the amount of text that is inside a link divided by the total text in the node.
|
||||
func getLinkDensity(s *goquery.Selection) float32 {
|
||||
linkLength := len(s.Find("a").Text())
|
||||
textLength := len(s.Text())
|
||||
|
||||
if textLength == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
return float32(linkLength) / float32(textLength)
|
||||
}
|
||||
|
||||
// Get an elements class/id weight. Uses regular expressions to tell if this
|
||||
// element looks good or bad.
|
||||
func getClassWeight(s *goquery.Selection) float32 {
|
||||
weight := 0
|
||||
class, _ := s.Attr("class")
|
||||
id, _ := s.Attr("id")
|
||||
|
||||
if class != "" {
|
||||
if negativeRegexp.MatchString(class) {
|
||||
weight -= 25
|
||||
}
|
||||
|
||||
if positiveRegexp.MatchString(class) {
|
||||
weight += 25
|
||||
}
|
||||
}
|
||||
|
||||
if id != "" {
|
||||
if negativeRegexp.MatchString(id) {
|
||||
weight -= 25
|
||||
}
|
||||
|
||||
if positiveRegexp.MatchString(id) {
|
||||
weight += 25
|
||||
}
|
||||
}
|
||||
|
||||
return float32(weight)
|
||||
}
|
||||
|
||||
func transformMisusedDivsIntoParagraphs(document *goquery.Document) {
|
||||
document.Find("div").Each(func(i int, s *goquery.Selection) {
|
||||
html, _ := s.Html()
|
||||
if !divToPElementsRegexp.MatchString(html) {
|
||||
node := s.Get(0)
|
||||
node.Data = "p"
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func removeNodes(s *goquery.Selection) {
|
||||
s.Each(func(i int, s *goquery.Selection) {
|
||||
parent := s.Parent()
|
||||
if parent.Length() > 0 {
|
||||
parent.Get(0).RemoveChild(s.Get(0))
|
||||
}
|
||||
})
|
||||
}
|
||||
38
reader/scraper/scraper.go
Normal file
38
reader/scraper/scraper.go
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
// Copyright 2017 Frédéric Guillot. All rights reserved.
|
||||
// Use of this source code is governed by the Apache 2.0
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package scraper
|
||||
|
||||
import (
|
||||
"errors"
|
||||
|
||||
"github.com/miniflux/miniflux2/http"
|
||||
"github.com/miniflux/miniflux2/reader/readability"
|
||||
"github.com/miniflux/miniflux2/reader/sanitizer"
|
||||
)
|
||||
|
||||
// Fetch download a web page a returns relevant contents.
|
||||
func Fetch(websiteURL string) (string, error) {
|
||||
client := http.NewClient(websiteURL)
|
||||
response, err := client.Get()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if response.HasServerFailure() {
|
||||
return "", errors.New("unable to download web page")
|
||||
}
|
||||
|
||||
page, err := response.NormalizeBodyEncoding()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
content, err := readability.ExtractContent(page)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return sanitizer.Sanitize(websiteURL, content), nil
|
||||
}
|
||||
|
|
@ -100,6 +100,7 @@ func getRoutes(cfg *config.Config, store *storage.Storage, feedHandler *feed.Han
|
|||
|
||||
router.Handle("/entry/status", uiHandler.Use(uiController.UpdateEntriesStatus)).Name("updateEntriesStatus").Methods("POST")
|
||||
router.Handle("/entry/save/{entryID}", uiHandler.Use(uiController.SaveEntry)).Name("saveEntry").Methods("POST")
|
||||
router.Handle("/entry/download/{entryID}", uiHandler.Use(uiController.FetchContent)).Name("fetchContent").Methods("POST")
|
||||
|
||||
router.Handle("/categories", uiHandler.Use(uiController.ShowCategories)).Name("categories").Methods("GET")
|
||||
router.Handle("/category/create", uiHandler.Use(uiController.CreateCategory)).Name("createCategory").Methods("GET")
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
// Code generated by go generate; DO NOT EDIT.
|
||||
// 2017-12-03 17:25:29.40151375 -0800 PST m=+0.014540675
|
||||
// 2017-12-10 18:56:24.36887959 -0800 PST m=+0.010858677
|
||||
|
||||
package static
|
||||
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -511,6 +511,14 @@ a.button {
|
|||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.entry-actions li {
|
||||
display: inline;
|
||||
}
|
||||
|
||||
.entry-actions li:not(:last-child):after {
|
||||
content: "|";
|
||||
}
|
||||
|
||||
.entry-meta {
|
||||
font-size: 0.95em;
|
||||
margin: 0 0 20px;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
// Code generated by go generate; DO NOT EDIT.
|
||||
// 2017-12-04 20:40:04.511740583 -0800 PST m=+0.012182340
|
||||
// 2017-12-10 18:56:24.37299237 -0800 PST m=+0.014971457
|
||||
|
||||
package static
|
||||
|
||||
|
|
@ -45,13 +45,16 @@ class EntryHandler{static updateEntriesStatus(entryIDs,status,callback){let url=
|
|||
static toggleEntryStatus(element){let entryID=parseInt(element.dataset.id,10);let statuses={read:"unread",unread:"read"};for(let currentStatus in statuses){let newStatus=statuses[currentStatus];if(element.classList.contains("item-status-"+currentStatus)){element.classList.remove("item-status-"+currentStatus);element.classList.add("item-status-"+newStatus);this.updateEntriesStatus([entryID],newStatus);break;}}}
|
||||
static markEntryAsRead(element){if(element.classList.contains("item-status-unread")){element.classList.remove("item-status-unread");element.classList.add("item-status-read");let entryID=parseInt(element.dataset.id,10);this.updateEntriesStatus([entryID],"read");}}
|
||||
static saveEntry(element){if(element.dataset.completed){return;}
|
||||
element.innerHTML=element.dataset.labelLoading;let request=new RequestBuilder(element.dataset.saveUrl);request.withCallback(()=>{element.innerHTML=element.dataset.labelDone;element.dataset.completed=true;});request.execute();}}
|
||||
element.innerHTML=element.dataset.labelLoading;let request=new RequestBuilder(element.dataset.saveUrl);request.withCallback(()=>{element.innerHTML=element.dataset.labelDone;element.dataset.completed=true;});request.execute();}
|
||||
static fetchOriginalContent(element){if(element.dataset.completed){return;}
|
||||
element.innerHTML=element.dataset.labelLoading;let request=new RequestBuilder(element.dataset.fetchContentUrl);request.withCallback((response)=>{element.innerHTML=element.dataset.labelDone;element.dataset.completed=true;response.json().then((data)=>{document.querySelector(".entry-content").innerHTML=data.content;});});request.execute();}}
|
||||
class ConfirmHandler{remove(url){let request=new RequestBuilder(url);request.withCallback(()=>window.location.reload());request.execute();}
|
||||
handle(event){let questionElement=document.createElement("span");let linkElement=event.target;let containerElement=linkElement.parentNode;linkElement.style.display="none";let yesElement=document.createElement("a");yesElement.href="#";yesElement.appendChild(document.createTextNode(linkElement.dataset.labelYes));yesElement.onclick=(event)=>{event.preventDefault();let loadingElement=document.createElement("span");loadingElement.className="loading";loadingElement.appendChild(document.createTextNode(linkElement.dataset.labelLoading));questionElement.remove();containerElement.appendChild(loadingElement);this.remove(linkElement.dataset.url);};let noElement=document.createElement("a");noElement.href="#";noElement.appendChild(document.createTextNode(linkElement.dataset.labelNo));noElement.onclick=(event)=>{event.preventDefault();linkElement.style.display="inline";questionElement.remove();};questionElement.className="confirm";questionElement.appendChild(document.createTextNode(linkElement.dataset.labelQuestion+" "));questionElement.appendChild(yesElement);questionElement.appendChild(document.createTextNode(", "));questionElement.appendChild(noElement);containerElement.appendChild(questionElement);}}
|
||||
class MenuHandler{clickMenuListItem(event){let element=event.target;if(element.tagName==="A"){window.location.href=element.getAttribute("href");}else{window.location.href=element.querySelector("a").getAttribute("href");}}
|
||||
toggleMainMenu(){let menu=document.querySelector(".header nav ul");if(DomHelper.isVisible(menu)){menu.style.display="none";}else{menu.style.display="block";}}}
|
||||
class NavHandler{markPageAsRead(){let items=DomHelper.getVisibleElements(".items .item");let entryIDs=[];items.forEach((element)=>{element.classList.add("item-status-read");entryIDs.push(parseInt(element.dataset.id,10));});if(entryIDs.length>0){EntryHandler.updateEntriesStatus(entryIDs,"read",()=>{this.goToPage("next",true);});}}
|
||||
saveEntry(){if(this.isListView()){let currentItem=document.querySelector(".current-item");if(currentItem!==null){let saveLink=currentItem.querySelector("a[data-save-entry]");if(saveLink){EntryHandler.saveEntry(saveLink);}}}else{let saveLink=document.querySelector("a[data-save-entry]");if(saveLink){EntryHandler.saveEntry(saveLink);}}}
|
||||
fetchOriginalContent(){if(!this.isListView()){let link=document.querySelector("a[data-fetch-content-entry]");if(link){EntryHandler.fetchOriginalContent(link);}}}
|
||||
toggleEntryStatus(){let currentItem=document.querySelector(".current-item");if(currentItem!==null){this.goToNextListItem();EntryHandler.toggleEntryStatus(currentItem);}}
|
||||
openOriginalLink(){let entryLink=document.querySelector(".entry h1 a");if(entryLink!==null){DomHelper.openNewTab(entryLink.getAttribute("href"));return;}
|
||||
let currentItemOriginalLink=document.querySelector(".current-item a[data-original-link]");if(currentItemOriginalLink!==null){DomHelper.openNewTab(currentItemOriginalLink.getAttribute("href"));let currentItem=document.querySelector(".current-item");this.goToNextListItem();EntryHandler.markEntryAsRead(currentItem);}}
|
||||
|
|
@ -68,9 +71,9 @@ if(currentItem===null){items[0].classList.add("current-item");return;}
|
|||
for(let i=0;i<items.length;i++){if(items[i].classList.contains("current-item")){items[i].classList.remove("current-item");if(i+1<items.length){items[i+1].classList.add("current-item");DomHelper.scrollPageTo(items[i+1]);}
|
||||
break;}}}
|
||||
isListView(){return document.querySelector(".items")!==null;}}
|
||||
document.addEventListener("DOMContentLoaded",function(){FormHandler.handleSubmitButtons();let touchHandler=new TouchHandler();touchHandler.listen();let navHandler=new NavHandler();let keyboardHandler=new KeyboardHandler();keyboardHandler.on("g u",()=>navHandler.goToPage("unread"));keyboardHandler.on("g h",()=>navHandler.goToPage("history"));keyboardHandler.on("g f",()=>navHandler.goToPage("feeds"));keyboardHandler.on("g c",()=>navHandler.goToPage("categories"));keyboardHandler.on("g s",()=>navHandler.goToPage("settings"));keyboardHandler.on("ArrowLeft",()=>navHandler.goToPrevious());keyboardHandler.on("ArrowRight",()=>navHandler.goToNext());keyboardHandler.on("j",()=>navHandler.goToPrevious());keyboardHandler.on("p",()=>navHandler.goToPrevious());keyboardHandler.on("k",()=>navHandler.goToNext());keyboardHandler.on("n",()=>navHandler.goToNext());keyboardHandler.on("h",()=>navHandler.goToPage("previous"));keyboardHandler.on("l",()=>navHandler.goToPage("next"));keyboardHandler.on("o",()=>navHandler.openSelectedItem());keyboardHandler.on("v",()=>navHandler.openOriginalLink());keyboardHandler.on("m",()=>navHandler.toggleEntryStatus());keyboardHandler.on("A",()=>navHandler.markPageAsRead());keyboardHandler.on("s",()=>navHandler.saveEntry());keyboardHandler.listen();let mouseHandler=new MouseHandler();mouseHandler.onClick("a[data-save-entry]",(event)=>{event.preventDefault();EntryHandler.saveEntry(event.target);});mouseHandler.onClick("a[data-on-click=markPageAsRead]",()=>navHandler.markPageAsRead());mouseHandler.onClick("a[data-confirm]",(event)=>{(new ConfirmHandler()).handle(event);});if(document.documentElement.clientWidth<600){let menuHandler=new MenuHandler();mouseHandler.onClick(".logo",()=>menuHandler.toggleMainMenu());mouseHandler.onClick(".header nav li",(event)=>menuHandler.clickMenuListItem(event));}});})();`,
|
||||
document.addEventListener("DOMContentLoaded",function(){FormHandler.handleSubmitButtons();let touchHandler=new TouchHandler();touchHandler.listen();let navHandler=new NavHandler();let keyboardHandler=new KeyboardHandler();keyboardHandler.on("g u",()=>navHandler.goToPage("unread"));keyboardHandler.on("g h",()=>navHandler.goToPage("history"));keyboardHandler.on("g f",()=>navHandler.goToPage("feeds"));keyboardHandler.on("g c",()=>navHandler.goToPage("categories"));keyboardHandler.on("g s",()=>navHandler.goToPage("settings"));keyboardHandler.on("ArrowLeft",()=>navHandler.goToPrevious());keyboardHandler.on("ArrowRight",()=>navHandler.goToNext());keyboardHandler.on("j",()=>navHandler.goToPrevious());keyboardHandler.on("p",()=>navHandler.goToPrevious());keyboardHandler.on("k",()=>navHandler.goToNext());keyboardHandler.on("n",()=>navHandler.goToNext());keyboardHandler.on("h",()=>navHandler.goToPage("previous"));keyboardHandler.on("l",()=>navHandler.goToPage("next"));keyboardHandler.on("o",()=>navHandler.openSelectedItem());keyboardHandler.on("v",()=>navHandler.openOriginalLink());keyboardHandler.on("m",()=>navHandler.toggleEntryStatus());keyboardHandler.on("A",()=>navHandler.markPageAsRead());keyboardHandler.on("s",()=>navHandler.saveEntry());keyboardHandler.on("d",()=>navHandler.fetchOriginalContent());keyboardHandler.listen();let mouseHandler=new MouseHandler();mouseHandler.onClick("a[data-save-entry]",(event)=>{event.preventDefault();EntryHandler.saveEntry(event.target);});mouseHandler.onClick("a[data-fetch-content-entry]",(event)=>{event.preventDefault();EntryHandler.fetchOriginalContent(event.target);});mouseHandler.onClick("a[data-on-click=markPageAsRead]",()=>navHandler.markPageAsRead());mouseHandler.onClick("a[data-confirm]",(event)=>{(new ConfirmHandler()).handle(event);});if(document.documentElement.clientWidth<600){let menuHandler=new MenuHandler();mouseHandler.onClick(".logo",()=>menuHandler.toggleMainMenu());mouseHandler.onClick(".header nav li",(event)=>menuHandler.clickMenuListItem(event));}});})();`,
|
||||
}
|
||||
|
||||
var JavascriptChecksums = map[string]string{
|
||||
"app": "7d14f00cb219662aaf59f20080265097eb236999dcc712b83775882970d05803",
|
||||
"app": "a70092cda52d5c3673e789868d8cfeb73a890e1a931b102a738021b5c2a65519",
|
||||
}
|
||||
|
|
|
|||
|
|
@ -324,6 +324,25 @@ class EntryHandler {
|
|||
});
|
||||
request.execute();
|
||||
}
|
||||
|
||||
static fetchOriginalContent(element) {
|
||||
if (element.dataset.completed) {
|
||||
return;
|
||||
}
|
||||
|
||||
element.innerHTML = element.dataset.labelLoading;
|
||||
|
||||
let request = new RequestBuilder(element.dataset.fetchContentUrl);
|
||||
request.withCallback((response) => {
|
||||
element.innerHTML = element.dataset.labelDone;
|
||||
element.dataset.completed = true;
|
||||
|
||||
response.json().then((data) => {
|
||||
document.querySelector(".entry-content").innerHTML = data.content;
|
||||
});
|
||||
});
|
||||
request.execute();
|
||||
}
|
||||
}
|
||||
|
||||
class ConfirmHandler {
|
||||
|
|
@ -430,6 +449,15 @@ class NavHandler {
|
|||
}
|
||||
}
|
||||
|
||||
fetchOriginalContent() {
|
||||
if (! this.isListView()){
|
||||
let link = document.querySelector("a[data-fetch-content-entry]");
|
||||
if (link) {
|
||||
EntryHandler.fetchOriginalContent(link);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
toggleEntryStatus() {
|
||||
let currentItem = document.querySelector(".current-item");
|
||||
if (currentItem !== null) {
|
||||
|
|
@ -577,6 +605,7 @@ document.addEventListener("DOMContentLoaded", function() {
|
|||
keyboardHandler.on("m", () => navHandler.toggleEntryStatus());
|
||||
keyboardHandler.on("A", () => navHandler.markPageAsRead());
|
||||
keyboardHandler.on("s", () => navHandler.saveEntry());
|
||||
keyboardHandler.on("d", () => navHandler.fetchOriginalContent());
|
||||
keyboardHandler.listen();
|
||||
|
||||
let mouseHandler = new MouseHandler();
|
||||
|
|
@ -584,6 +613,12 @@ document.addEventListener("DOMContentLoaded", function() {
|
|||
event.preventDefault();
|
||||
EntryHandler.saveEntry(event.target);
|
||||
});
|
||||
|
||||
mouseHandler.onClick("a[data-fetch-content-entry]", (event) => {
|
||||
event.preventDefault();
|
||||
EntryHandler.fetchOriginalContent(event.target);
|
||||
});
|
||||
|
||||
mouseHandler.onClick("a[data-on-click=markPageAsRead]", () => navHandler.markPageAsRead());
|
||||
mouseHandler.onClick("a[data-confirm]", (event) => {
|
||||
(new ConfirmHandler()).handle(event);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
// Code generated by go generate; DO NOT EDIT.
|
||||
// 2017-12-03 17:25:29.427766854 -0800 PST m=+0.040793779
|
||||
// 2017-12-10 18:56:24.386027486 -0800 PST m=+0.028006573
|
||||
|
||||
package template
|
||||
|
||||
|
|
|
|||
|
|
@ -7,13 +7,26 @@
|
|||
<a href="{{ .entry.URL }}" target="_blank" rel="noopener noreferrer" referrerpolicy="no-referrer">{{ .entry.Title }}</a>
|
||||
</h1>
|
||||
<div class="entry-actions">
|
||||
<a href="#"
|
||||
title="{{ t "Save this article" }}"
|
||||
data-save-entry="true"
|
||||
data-save-url="{{ route "saveEntry" "entryID" .entry.ID }}"
|
||||
data-label-loading="{{ t "Saving..." }}"
|
||||
data-label-done="{{ t "Done!" }}"
|
||||
>{{ t "Save" }}</a>
|
||||
<ul>
|
||||
<li>
|
||||
<a href="#"
|
||||
title="{{ t "Save this article" }}"
|
||||
data-save-entry="true"
|
||||
data-save-url="{{ route "saveEntry" "entryID" .entry.ID }}"
|
||||
data-label-loading="{{ t "Saving..." }}"
|
||||
data-label-done="{{ t "Done!" }}"
|
||||
>{{ t "Save" }}</a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="#"
|
||||
title="{{ t "Fetch original content" }}"
|
||||
data-fetch-content-entry="true"
|
||||
data-fetch-content-url="{{ route "fetchContent" "entryID" .entry.ID }}"
|
||||
data-label-loading="{{ t "Loading..." }}"
|
||||
data-label-done="{{ t "Done!" }}"
|
||||
>{{ t "Fetch original content" }}</a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="entry-meta">
|
||||
<span class="entry-website">
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
// Code generated by go generate; DO NOT EDIT.
|
||||
// 2017-12-04 20:56:07.05263963 -0800 PST m=+0.018799946
|
||||
// 2017-12-10 18:56:24.375327888 -0800 PST m=+0.017306975
|
||||
|
||||
package template
|
||||
|
||||
|
|
@ -466,13 +466,26 @@ var templateViewsMap = map[string]string{
|
|||
<a href="{{ .entry.URL }}" target="_blank" rel="noopener noreferrer" referrerpolicy="no-referrer">{{ .entry.Title }}</a>
|
||||
</h1>
|
||||
<div class="entry-actions">
|
||||
<a href="#"
|
||||
title="{{ t "Save this article" }}"
|
||||
data-save-entry="true"
|
||||
data-save-url="{{ route "saveEntry" "entryID" .entry.ID }}"
|
||||
data-label-loading="{{ t "Saving..." }}"
|
||||
data-label-done="{{ t "Done!" }}"
|
||||
>{{ t "Save" }}</a>
|
||||
<ul>
|
||||
<li>
|
||||
<a href="#"
|
||||
title="{{ t "Save this article" }}"
|
||||
data-save-entry="true"
|
||||
data-save-url="{{ route "saveEntry" "entryID" .entry.ID }}"
|
||||
data-label-loading="{{ t "Saving..." }}"
|
||||
data-label-done="{{ t "Done!" }}"
|
||||
>{{ t "Save" }}</a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="#"
|
||||
title="{{ t "Fetch original content" }}"
|
||||
data-fetch-content-entry="true"
|
||||
data-fetch-content-url="{{ route "fetchContent" "entryID" .entry.ID }}"
|
||||
data-label-loading="{{ t "Loading..." }}"
|
||||
data-label-done="{{ t "Done!" }}"
|
||||
>{{ t "Fetch original content" }}</a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
<div class="entry-meta">
|
||||
<span class="entry-website">
|
||||
|
|
@ -1170,7 +1183,7 @@ var templateViewsMapChecksums = map[string]string{
|
|||
"edit_category": "cee720faadcec58289b707ad30af623d2ee66c1ce23a732965463250d7ff41c5",
|
||||
"edit_feed": "c5bc4c22bf7e8348d880395250545595d21fb8c8e723fc5d7cca68e25d250884",
|
||||
"edit_user": "82d9749d76ddbd2352816d813c4b1f6d92f2222de678b4afe5821090246735c7",
|
||||
"entry": "7b234e551a98233d9797948db8a000e3d10334e17d5b1d5d17552d1406555b34",
|
||||
"entry": "ebcf9bb35812dd02759718f7f7411267e6a6c8efd59a9aa0a0e735bcb88efeff",
|
||||
"feed_entries": "547c19eb36b20e350ce70ed045173b064cdcd6b114afb241c9f2dda9d88fcc27",
|
||||
"feeds": "c22af39b42ba9ca69ea0914ca789303ec2c5b484abcd4eaa49016e365381257c",
|
||||
"history": "9a67599a5d8d67ef958e3f07da339b749f42892667547c9e60a54477e8d32a56",
|
||||
|
|
|
|||
|
|
@ -8,12 +8,91 @@ import (
|
|||
"errors"
|
||||
"log"
|
||||
|
||||
"github.com/miniflux/miniflux2/integration"
|
||||
"github.com/miniflux/miniflux2/model"
|
||||
"github.com/miniflux/miniflux2/reader/scraper"
|
||||
"github.com/miniflux/miniflux2/server/core"
|
||||
"github.com/miniflux/miniflux2/server/ui/payload"
|
||||
"github.com/miniflux/miniflux2/storage"
|
||||
)
|
||||
|
||||
// FetchContent downloads the original HTML page and returns relevant contents.
|
||||
func (c *Controller) FetchContent(ctx *core.Context, request *core.Request, response *core.Response) {
|
||||
entryID, err := request.IntegerParam("entryID")
|
||||
if err != nil {
|
||||
response.HTML().BadRequest(err)
|
||||
return
|
||||
}
|
||||
|
||||
user := ctx.LoggedUser()
|
||||
builder := c.store.GetEntryQueryBuilder(user.ID, user.Timezone)
|
||||
builder.WithEntryID(entryID)
|
||||
builder.WithoutStatus(model.EntryStatusRemoved)
|
||||
|
||||
entry, err := builder.GetEntry()
|
||||
if err != nil {
|
||||
response.JSON().ServerError(err)
|
||||
return
|
||||
}
|
||||
|
||||
if entry == nil {
|
||||
response.JSON().NotFound(errors.New("Entry not found"))
|
||||
return
|
||||
}
|
||||
|
||||
content, err := scraper.Fetch(entry.URL)
|
||||
if err != nil {
|
||||
response.JSON().ServerError(err)
|
||||
return
|
||||
}
|
||||
|
||||
if len(content) > len(entry.Content) {
|
||||
entry.Content = content
|
||||
c.store.UpdateEntryContent(entry)
|
||||
} else {
|
||||
content = entry.Content
|
||||
}
|
||||
|
||||
response.JSON().Created(map[string]string{"content": content})
|
||||
}
|
||||
|
||||
// SaveEntry send the link to external services.
|
||||
func (c *Controller) SaveEntry(ctx *core.Context, request *core.Request, response *core.Response) {
|
||||
entryID, err := request.IntegerParam("entryID")
|
||||
if err != nil {
|
||||
response.HTML().BadRequest(err)
|
||||
return
|
||||
}
|
||||
|
||||
user := ctx.LoggedUser()
|
||||
builder := c.store.GetEntryQueryBuilder(user.ID, user.Timezone)
|
||||
builder.WithEntryID(entryID)
|
||||
builder.WithoutStatus(model.EntryStatusRemoved)
|
||||
|
||||
entry, err := builder.GetEntry()
|
||||
if err != nil {
|
||||
response.JSON().ServerError(err)
|
||||
return
|
||||
}
|
||||
|
||||
if entry == nil {
|
||||
response.JSON().NotFound(errors.New("Entry not found"))
|
||||
return
|
||||
}
|
||||
|
||||
settings, err := c.store.Integration(user.ID)
|
||||
if err != nil {
|
||||
response.JSON().ServerError(err)
|
||||
return
|
||||
}
|
||||
|
||||
go func() {
|
||||
integration.SendEntry(entry, settings)
|
||||
}()
|
||||
|
||||
response.JSON().Created(map[string]string{"message": "saved"})
|
||||
}
|
||||
|
||||
// ShowFeedEntry shows a single feed entry in "feed" mode.
|
||||
func (c *Controller) ShowFeedEntry(ctx *core.Context, request *core.Request, response *core.Response) {
|
||||
user := ctx.LoggedUser()
|
||||
|
|
|
|||
|
|
@ -6,11 +6,8 @@ package controller
|
|||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/miniflux/miniflux2/integration"
|
||||
"github.com/miniflux/miniflux2/model"
|
||||
"github.com/miniflux/miniflux2/server/core"
|
||||
"github.com/miniflux/miniflux2/server/ui/form"
|
||||
)
|
||||
|
|
@ -73,40 +70,3 @@ func (c *Controller) UpdateIntegration(ctx *core.Context, request *core.Request,
|
|||
|
||||
response.Redirect(ctx.Route("integrations"))
|
||||
}
|
||||
|
||||
// SaveEntry send the link to external services.
|
||||
func (c *Controller) SaveEntry(ctx *core.Context, request *core.Request, response *core.Response) {
|
||||
entryID, err := request.IntegerParam("entryID")
|
||||
if err != nil {
|
||||
response.HTML().BadRequest(err)
|
||||
return
|
||||
}
|
||||
|
||||
user := ctx.LoggedUser()
|
||||
builder := c.store.GetEntryQueryBuilder(user.ID, user.Timezone)
|
||||
builder.WithEntryID(entryID)
|
||||
builder.WithoutStatus(model.EntryStatusRemoved)
|
||||
|
||||
entry, err := builder.GetEntry()
|
||||
if err != nil {
|
||||
response.JSON().ServerError(err)
|
||||
return
|
||||
}
|
||||
|
||||
if entry == nil {
|
||||
response.JSON().NotFound(errors.New("Entry not found"))
|
||||
return
|
||||
}
|
||||
|
||||
settings, err := c.store.Integration(user.ID)
|
||||
if err != nil {
|
||||
response.JSON().ServerError(err)
|
||||
return
|
||||
}
|
||||
|
||||
go func() {
|
||||
integration.SendEntry(entry, settings)
|
||||
}()
|
||||
|
||||
response.JSON().Created(map[string]string{"message": "saved"})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
// Code generated by go generate; DO NOT EDIT.
|
||||
// 2017-12-03 17:25:29.391052668 -0800 PST m=+0.004079593
|
||||
// 2017-12-10 18:56:24.36359961 -0800 PST m=+0.005578697
|
||||
|
||||
package sql
|
||||
|
||||
|
|
|
|||
|
|
@ -59,6 +59,23 @@ func (s *Storage) CreateEntry(entry *model.Entry) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// UpdateEntryContent updates entry content.
|
||||
func (s *Storage) UpdateEntryContent(entry *model.Entry) error {
|
||||
query := `
|
||||
UPDATE entries SET
|
||||
content=$1
|
||||
WHERE user_id=$2 AND id=$3
|
||||
`
|
||||
|
||||
_, err := s.db.Exec(
|
||||
query,
|
||||
entry.Content,
|
||||
entry.UserID,
|
||||
entry.ID,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
// UpdateEntry update an entry when a feed is refreshed.
|
||||
func (s *Storage) UpdateEntry(entry *model.Entry) error {
|
||||
query := `
|
||||
|
|
|
|||
Loading…
Reference in a new issue