Add readability package to fetch original content

locale/translations.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2017-12-04 21:22:23.44799753 -0800 PST m=+0.006646042
+// 2017-12-10 18:56:24.387844114 -0800 PST m=+0.029823201
 
 package locale
 
@@ -166,12 +166,13 @@ var translations = map[string]string{
     "Instapaper Password": "Mot de passe Instapaper",
     "Activate Fever API": "Activer l'API de Fever",
     "Fever Username": "Nom d'utilisateur pour l'API de Fever",
-    "Fever Password": "Mot de passe pour l'API de Fever"
+    "Fever Password": "Mot de passe pour l'API de Fever",
+    "Fetch original content": "Récupérer le contenu original"
 }
 `,
 }
 
 var translationsChecksums = map[string]string{
 	"en_US": "6fe95384260941e8a5a3c695a655a932e0a8a6a572c1e45cb2b1ae8baa01b897",
-	"fr_FR": "ef3d095f3e78d88a2746240769fa30d2e83c6519187d98e2193c9231dda5d882",
+	"fr_FR": "fd629b171aefa50dd0a6100acaac8fbecbdf1a1d53e3fce984234565ec5bb5d5",
 }

locale/translations/fr_FR.json

@@ -150,5 +150,6 @@
     "Instapaper Password": "Mot de passe Instapaper",
     "Activate Fever API": "Activer l'API de Fever",
     "Fever Username": "Nom d'utilisateur pour l'API de Fever",
-    "Fever Password": "Mot de passe pour l'API de Fever"
+    "Fever Password": "Mot de passe pour l'API de Fever",
+    "Fetch original content": "Récupérer le contenu original"
 }

reader/readability/readability.go

@@ -0,0 +1,306 @@
+// Copyright 2017 Frédéric Guillot. All rights reserved.
+// Use of this source code is governed by the Apache 2.0
+// license that can be found in the LICENSE file.
+
+package readability
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"math"
+	"regexp"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+	"golang.org/x/net/html"
+)
+
+const (
+	defaultTagsToScore = "section,h2,h3,h4,h5,h6,p,td,pre,div"
+)
+
+var (
+	divToPElementsRegexp = regexp.MustCompile(`(?i)<(a|blockquote|dl|div|img|ol|p|pre|table|ul)`)
+	sentenceRegexp       = regexp.MustCompile(`\.( |$)`)
+
+	blacklistCandidatesRegexp  = regexp.MustCompile(`(?i)popupbody|-ad|g-plus`)
+	okMaybeItsACandidateRegexp = regexp.MustCompile(`(?i)and|article|body|column|main|shadow`)
+	unlikelyCandidatesRegexp   = regexp.MustCompile(`(?i)banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|foot|header|legends|menu|modal|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote`)
+
+	negativeRegexp = regexp.MustCompile(`(?i)hidden|^hid$|hid$|hid|^hid |banner|combx|comment|com-|contact|foot|footer|footnote|masthead|media|meta|modal|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|tool|widget|byline|author|dateline|writtenby|p-author`)
+	positiveRegexp = regexp.MustCompile(`(?i)article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story`)
+)
+
+type candidate struct {
+	selection *goquery.Selection
+	score     float32
+}
+
+func (c *candidate) Node() *html.Node {
+	return c.selection.Get(0)
+}
+
+func (c *candidate) String() string {
+	id, _ := c.selection.Attr("id")
+	class, _ := c.selection.Attr("class")
+
+	if id != "" && class != "" {
+		return fmt.Sprintf("%s#%s.%s => %f", c.Node().DataAtom, id, class, c.score)
+	} else if id != "" {
+		return fmt.Sprintf("%s#%s => %f", c.Node().DataAtom, id, c.score)
+	} else if class != "" {
+		return fmt.Sprintf("%s.%s => %f", c.Node().DataAtom, class, c.score)
+	}
+
+	return fmt.Sprintf("%s => %f", c.Node().DataAtom, c.score)
+}
+
+type candidateList map[*html.Node]*candidate
+
+func (c candidateList) String() string {
+	var output []string
+	for _, candidate := range c {
+		output = append(output, candidate.String())
+	}
+
+	return strings.Join(output, ", ")
+}
+
+// ExtractContent returns relevant content.
+func ExtractContent(page io.Reader) (string, error) {
+	document, err := goquery.NewDocumentFromReader(page)
+	if err != nil {
+		return "", err
+	}
+
+	document.Find("script,style,noscript").Each(func(i int, s *goquery.Selection) {
+		removeNodes(s)
+	})
+
+	transformMisusedDivsIntoParagraphs(document)
+	removeUnlikelyCandidates(document)
+
+	candidates := getCandidates(document)
+	log.Println("Candidates:", candidates)
+
+	topCandidate := getTopCandidate(document, candidates)
+	log.Println("TopCandidate:", topCandidate)
+
+	output := getArticle(topCandidate, candidates)
+	return output, nil
+}
+
+// Now that we have the top candidate, look through its siblings for content that might also be related.
+// Things like preambles, content split by ads that we removed, etc.
+func getArticle(topCandidate *candidate, candidates candidateList) string {
+	output := bytes.NewBufferString("<div>")
+	siblingScoreThreshold := float32(math.Max(10, float64(topCandidate.score*.2)))
+
+	topCandidate.selection.Siblings().Union(topCandidate.selection).Each(func(i int, s *goquery.Selection) {
+		append := false
+		node := s.Get(0)
+
+		if node == topCandidate.Node() {
+			append = true
+		} else if c, ok := candidates[node]; ok && c.score >= siblingScoreThreshold {
+			append = true
+		}
+
+		if s.Is("p") {
+			linkDensity := getLinkDensity(s)
+			content := s.Text()
+			contentLength := len(content)
+
+			if contentLength >= 80 && linkDensity < .25 {
+				append = true
+			} else if contentLength < 80 && linkDensity == 0 && sentenceRegexp.MatchString(content) {
+				append = true
+			}
+		}
+
+		if append {
+			tag := "div"
+			if s.Is("p") {
+				tag = node.Data
+			}
+
+			html, _ := s.Html()
+			fmt.Fprintf(output, "<%s>%s</%s>", tag, html, tag)
+		}
+	})
+
+	output.Write([]byte("</div>"))
+	return output.String()
+}
+
+func removeUnlikelyCandidates(document *goquery.Document) {
+	document.Find("*").Not("html,body").Each(func(i int, s *goquery.Selection) {
+		class, _ := s.Attr("class")
+		id, _ := s.Attr("id")
+		str := class + id
+
+		if blacklistCandidatesRegexp.MatchString(str) || (unlikelyCandidatesRegexp.MatchString(str) && !okMaybeItsACandidateRegexp.MatchString(str)) {
+			// log.Printf("Removing unlikely candidate - %s\n", str)
+			removeNodes(s)
+		}
+	})
+}
+
+func getTopCandidate(document *goquery.Document, candidates candidateList) *candidate {
+	var best *candidate
+
+	for _, c := range candidates {
+		if best == nil {
+			best = c
+		} else if best.score < c.score {
+			best = c
+		}
+	}
+
+	if best == nil {
+		best = &candidate{document.Find("body"), 0}
+	}
+
+	return best
+}
+
+// Loop through all paragraphs, and assign a score to them based on how content-y they look.
+// Then add their score to their parent node.
+// A score is determined by things like number of commas, class names, etc.
+// Maybe eventually link density.
+func getCandidates(document *goquery.Document) candidateList {
+	candidates := make(candidateList)
+
+	document.Find(defaultTagsToScore).Each(func(i int, s *goquery.Selection) {
+		text := s.Text()
+
+		// If this paragraph is less than 25 characters, don't even count it.
+		if len(text) < 25 {
+			return
+		}
+
+		parent := s.Parent()
+		parentNode := parent.Get(0)
+
+		grandParent := parent.Parent()
+		var grandParentNode *html.Node
+		if grandParent.Length() > 0 {
+			grandParentNode = grandParent.Get(0)
+		}
+
+		if _, found := candidates[parentNode]; !found {
+			candidates[parentNode] = scoreNode(parent)
+		}
+
+		if grandParentNode != nil {
+			if _, found := candidates[grandParentNode]; !found {
+				candidates[grandParentNode] = scoreNode(grandParent)
+			}
+		}
+
+		// Add a point for the paragraph itself as a base.
+		contentScore := float32(1.0)
+
+		// Add points for any commas within this paragraph.
+		contentScore += float32(strings.Count(text, ",") + 1)
+
+		// For every 100 characters in this paragraph, add another point. Up to 3 points.
+		contentScore += float32(math.Min(float64(int(len(text)/100.0)), 3))
+
+		candidates[parentNode].score += contentScore
+		if grandParentNode != nil {
+			candidates[grandParentNode].score += contentScore / 2.0
+		}
+	})
+
+	// Scale the final candidates score based on link density. Good content
+	// should have a relatively small link density (5% or less) and be mostly
+	// unaffected by this operation
+	for _, candidate := range candidates {
+		candidate.score = candidate.score * (1 - getLinkDensity(candidate.selection))
+	}
+
+	return candidates
+}
+
+func scoreNode(s *goquery.Selection) *candidate {
+	c := &candidate{selection: s, score: 0}
+
+	switch s.Get(0).DataAtom.String() {
+	case "div":
+		c.score += 5
+	case "pre", "td", "blockquote", "img":
+		c.score += 3
+	case "address", "ol", "ul", "dl", "dd", "dt", "li", "form":
+		c.score -= 3
+	case "h1", "h2", "h3", "h4", "h5", "h6", "th":
+		c.score -= 5
+	}
+
+	c.score += getClassWeight(s)
+	return c
+}
+
+// Get the density of links as a percentage of the content
+// This is the amount of text that is inside a link divided by the total text in the node.
+func getLinkDensity(s *goquery.Selection) float32 {
+	linkLength := len(s.Find("a").Text())
+	textLength := len(s.Text())
+
+	if textLength == 0 {
+		return 0
+	}
+
+	return float32(linkLength) / float32(textLength)
+}
+
+// Get an elements class/id weight. Uses regular expressions to tell if this
+// element looks good or bad.
+func getClassWeight(s *goquery.Selection) float32 {
+	weight := 0
+	class, _ := s.Attr("class")
+	id, _ := s.Attr("id")
+
+	if class != "" {
+		if negativeRegexp.MatchString(class) {
+			weight -= 25
+		}
+
+		if positiveRegexp.MatchString(class) {
+			weight += 25
+		}
+	}
+
+	if id != "" {
+		if negativeRegexp.MatchString(id) {
+			weight -= 25
+		}
+
+		if positiveRegexp.MatchString(id) {
+			weight += 25
+		}
+	}
+
+	return float32(weight)
+}
+
+func transformMisusedDivsIntoParagraphs(document *goquery.Document) {
+	document.Find("div").Each(func(i int, s *goquery.Selection) {
+		html, _ := s.Html()
+		if !divToPElementsRegexp.MatchString(html) {
+			node := s.Get(0)
+			node.Data = "p"
+		}
+	})
+}
+
+func removeNodes(s *goquery.Selection) {
+	s.Each(func(i int, s *goquery.Selection) {
+		parent := s.Parent()
+		if parent.Length() > 0 {
+			parent.Get(0).RemoveChild(s.Get(0))
+		}
+	})
+}

reader/scraper/scraper.go

@@ -0,0 +1,38 @@
+// Copyright 2017 Frédéric Guillot. All rights reserved.
+// Use of this source code is governed by the Apache 2.0
+// license that can be found in the LICENSE file.
+
+package scraper
+
+import (
+	"errors"
+
+	"github.com/miniflux/miniflux2/http"
+	"github.com/miniflux/miniflux2/reader/readability"
+	"github.com/miniflux/miniflux2/reader/sanitizer"
+)
+
+// Fetch download a web page a returns relevant contents.
+func Fetch(websiteURL string) (string, error) {
+	client := http.NewClient(websiteURL)
+	response, err := client.Get()
+	if err != nil {
+		return "", err
+	}
+
+	if response.HasServerFailure() {
+		return "", errors.New("unable to download web page")
+	}
+
+	page, err := response.NormalizeBodyEncoding()
+	if err != nil {
+		return "", err
+	}
+
+	content, err := readability.ExtractContent(page)
+	if err != nil {
+		return "", err
+	}
+
+	return sanitizer.Sanitize(websiteURL, content), nil
+}

server/routes.go

@@ -100,6 +100,7 @@ func getRoutes(cfg *config.Config, store *storage.Storage, feedHandler *feed.Han
 
 	router.Handle("/entry/status", uiHandler.Use(uiController.UpdateEntriesStatus)).Name("updateEntriesStatus").Methods("POST")
 	router.Handle("/entry/save/{entryID}", uiHandler.Use(uiController.SaveEntry)).Name("saveEntry").Methods("POST")
+	router.Handle("/entry/download/{entryID}", uiHandler.Use(uiController.FetchContent)).Name("fetchContent").Methods("POST")
 
 	router.Handle("/categories", uiHandler.Use(uiController.ShowCategories)).Name("categories").Methods("GET")
 	router.Handle("/category/create", uiHandler.Use(uiController.CreateCategory)).Name("createCategory").Methods("GET")

server/static/bin.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2017-12-03 17:25:29.40151375 -0800 PST m=+0.014540675
+// 2017-12-10 18:56:24.36887959 -0800 PST m=+0.010858677
 
 package static
 

server/static/css/common.css

@@ -511,6 +511,14 @@ a.button {
     margin-bottom: 20px;
 }
 
+.entry-actions li {
+    display: inline;
+}
+
+.entry-actions li:not(:last-child):after {
+    content: "|";
+}
+
 .entry-meta {
     font-size: 0.95em;
     margin: 0 0 20px;

server/static/js.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2017-12-04 20:40:04.511740583 -0800 PST m=+0.012182340
+// 2017-12-10 18:56:24.37299237 -0800 PST m=+0.014971457
 
 package static
 
@@ -45,13 +45,16 @@ class EntryHandler{static updateEntriesStatus(entryIDs,status,callback){let url=
 static toggleEntryStatus(element){let entryID=parseInt(element.dataset.id,10);let statuses={read:"unread",unread:"read"};for(let currentStatus in statuses){let newStatus=statuses[currentStatus];if(element.classList.contains("item-status-"+currentStatus)){element.classList.remove("item-status-"+currentStatus);element.classList.add("item-status-"+newStatus);this.updateEntriesStatus([entryID],newStatus);break;}}}
 static markEntryAsRead(element){if(element.classList.contains("item-status-unread")){element.classList.remove("item-status-unread");element.classList.add("item-status-read");let entryID=parseInt(element.dataset.id,10);this.updateEntriesStatus([entryID],"read");}}
 static saveEntry(element){if(element.dataset.completed){return;}
-element.innerHTML=element.dataset.labelLoading;let request=new RequestBuilder(element.dataset.saveUrl);request.withCallback(()=>{element.innerHTML=element.dataset.labelDone;element.dataset.completed=true;});request.execute();}}
+element.innerHTML=element.dataset.labelLoading;let request=new RequestBuilder(element.dataset.saveUrl);request.withCallback(()=>{element.innerHTML=element.dataset.labelDone;element.dataset.completed=true;});request.execute();}
+static fetchOriginalContent(element){if(element.dataset.completed){return;}
+element.innerHTML=element.dataset.labelLoading;let request=new RequestBuilder(element.dataset.fetchContentUrl);request.withCallback((response)=>{element.innerHTML=element.dataset.labelDone;element.dataset.completed=true;response.json().then((data)=>{document.querySelector(".entry-content").innerHTML=data.content;});});request.execute();}}
 class ConfirmHandler{remove(url){let request=new RequestBuilder(url);request.withCallback(()=>window.location.reload());request.execute();}
 handle(event){let questionElement=document.createElement("span");let linkElement=event.target;let containerElement=linkElement.parentNode;linkElement.style.display="none";let yesElement=document.createElement("a");yesElement.href="#";yesElement.appendChild(document.createTextNode(linkElement.dataset.labelYes));yesElement.onclick=(event)=>{event.preventDefault();let loadingElement=document.createElement("span");loadingElement.className="loading";loadingElement.appendChild(document.createTextNode(linkElement.dataset.labelLoading));questionElement.remove();containerElement.appendChild(loadingElement);this.remove(linkElement.dataset.url);};let noElement=document.createElement("a");noElement.href="#";noElement.appendChild(document.createTextNode(linkElement.dataset.labelNo));noElement.onclick=(event)=>{event.preventDefault();linkElement.style.display="inline";questionElement.remove();};questionElement.className="confirm";questionElement.appendChild(document.createTextNode(linkElement.dataset.labelQuestion+" "));questionElement.appendChild(yesElement);questionElement.appendChild(document.createTextNode(", "));questionElement.appendChild(noElement);containerElement.appendChild(questionElement);}}
 class MenuHandler{clickMenuListItem(event){let element=event.target;if(element.tagName==="A"){window.location.href=element.getAttribute("href");}else{window.location.href=element.querySelector("a").getAttribute("href");}}
 toggleMainMenu(){let menu=document.querySelector(".header nav ul");if(DomHelper.isVisible(menu)){menu.style.display="none";}else{menu.style.display="block";}}}
 class NavHandler{markPageAsRead(){let items=DomHelper.getVisibleElements(".items .item");let entryIDs=[];items.forEach((element)=>{element.classList.add("item-status-read");entryIDs.push(parseInt(element.dataset.id,10));});if(entryIDs.length>0){EntryHandler.updateEntriesStatus(entryIDs,"read",()=>{this.goToPage("next",true);});}}
 saveEntry(){if(this.isListView()){let currentItem=document.querySelector(".current-item");if(currentItem!==null){let saveLink=currentItem.querySelector("a[data-save-entry]");if(saveLink){EntryHandler.saveEntry(saveLink);}}}else{let saveLink=document.querySelector("a[data-save-entry]");if(saveLink){EntryHandler.saveEntry(saveLink);}}}
+fetchOriginalContent(){if(!this.isListView()){let link=document.querySelector("a[data-fetch-content-entry]");if(link){EntryHandler.fetchOriginalContent(link);}}}
 toggleEntryStatus(){let currentItem=document.querySelector(".current-item");if(currentItem!==null){this.goToNextListItem();EntryHandler.toggleEntryStatus(currentItem);}}
 openOriginalLink(){let entryLink=document.querySelector(".entry h1 a");if(entryLink!==null){DomHelper.openNewTab(entryLink.getAttribute("href"));return;}
 let currentItemOriginalLink=document.querySelector(".current-item a[data-original-link]");if(currentItemOriginalLink!==null){DomHelper.openNewTab(currentItemOriginalLink.getAttribute("href"));let currentItem=document.querySelector(".current-item");this.goToNextListItem();EntryHandler.markEntryAsRead(currentItem);}}
@@ -68,9 +71,9 @@ if(currentItem===null){items[0].classList.add("current-item");return;}
 for(let i=0;i<items.length;i++){if(items[i].classList.contains("current-item")){items[i].classList.remove("current-item");if(i+1<items.length){items[i+1].classList.add("current-item");DomHelper.scrollPageTo(items[i+1]);}
 break;}}}
 isListView(){return document.querySelector(".items")!==null;}}
-document.addEventListener("DOMContentLoaded",function(){FormHandler.handleSubmitButtons();let touchHandler=new TouchHandler();touchHandler.listen();let navHandler=new NavHandler();let keyboardHandler=new KeyboardHandler();keyboardHandler.on("g u",()=>navHandler.goToPage("unread"));keyboardHandler.on("g h",()=>navHandler.goToPage("history"));keyboardHandler.on("g f",()=>navHandler.goToPage("feeds"));keyboardHandler.on("g c",()=>navHandler.goToPage("categories"));keyboardHandler.on("g s",()=>navHandler.goToPage("settings"));keyboardHandler.on("ArrowLeft",()=>navHandler.goToPrevious());keyboardHandler.on("ArrowRight",()=>navHandler.goToNext());keyboardHandler.on("j",()=>navHandler.goToPrevious());keyboardHandler.on("p",()=>navHandler.goToPrevious());keyboardHandler.on("k",()=>navHandler.goToNext());keyboardHandler.on("n",()=>navHandler.goToNext());keyboardHandler.on("h",()=>navHandler.goToPage("previous"));keyboardHandler.on("l",()=>navHandler.goToPage("next"));keyboardHandler.on("o",()=>navHandler.openSelectedItem());keyboardHandler.on("v",()=>navHandler.openOriginalLink());keyboardHandler.on("m",()=>navHandler.toggleEntryStatus());keyboardHandler.on("A",()=>navHandler.markPageAsRead());keyboardHandler.on("s",()=>navHandler.saveEntry());keyboardHandler.listen();let mouseHandler=new MouseHandler();mouseHandler.onClick("a[data-save-entry]",(event)=>{event.preventDefault();EntryHandler.saveEntry(event.target);});mouseHandler.onClick("a[data-on-click=markPageAsRead]",()=>navHandler.markPageAsRead());mouseHandler.onClick("a[data-confirm]",(event)=>{(new ConfirmHandler()).handle(event);});if(document.documentElement.clientWidth<600){let menuHandler=new MenuHandler();mouseHandler.onClick(".logo",()=>menuHandler.toggleMainMenu());mouseHandler.onClick(".header nav li",(event)=>menuHandler.clickMenuListItem(event));}});})();`,
+document.addEventListener("DOMContentLoaded",function(){FormHandler.handleSubmitButtons();let touchHandler=new TouchHandler();touchHandler.listen();let navHandler=new NavHandler();let keyboardHandler=new KeyboardHandler();keyboardHandler.on("g u",()=>navHandler.goToPage("unread"));keyboardHandler.on("g h",()=>navHandler.goToPage("history"));keyboardHandler.on("g f",()=>navHandler.goToPage("feeds"));keyboardHandler.on("g c",()=>navHandler.goToPage("categories"));keyboardHandler.on("g s",()=>navHandler.goToPage("settings"));keyboardHandler.on("ArrowLeft",()=>navHandler.goToPrevious());keyboardHandler.on("ArrowRight",()=>navHandler.goToNext());keyboardHandler.on("j",()=>navHandler.goToPrevious());keyboardHandler.on("p",()=>navHandler.goToPrevious());keyboardHandler.on("k",()=>navHandler.goToNext());keyboardHandler.on("n",()=>navHandler.goToNext());keyboardHandler.on("h",()=>navHandler.goToPage("previous"));keyboardHandler.on("l",()=>navHandler.goToPage("next"));keyboardHandler.on("o",()=>navHandler.openSelectedItem());keyboardHandler.on("v",()=>navHandler.openOriginalLink());keyboardHandler.on("m",()=>navHandler.toggleEntryStatus());keyboardHandler.on("A",()=>navHandler.markPageAsRead());keyboardHandler.on("s",()=>navHandler.saveEntry());keyboardHandler.on("d",()=>navHandler.fetchOriginalContent());keyboardHandler.listen();let mouseHandler=new MouseHandler();mouseHandler.onClick("a[data-save-entry]",(event)=>{event.preventDefault();EntryHandler.saveEntry(event.target);});mouseHandler.onClick("a[data-fetch-content-entry]",(event)=>{event.preventDefault();EntryHandler.fetchOriginalContent(event.target);});mouseHandler.onClick("a[data-on-click=markPageAsRead]",()=>navHandler.markPageAsRead());mouseHandler.onClick("a[data-confirm]",(event)=>{(new ConfirmHandler()).handle(event);});if(document.documentElement.clientWidth<600){let menuHandler=new MenuHandler();mouseHandler.onClick(".logo",()=>menuHandler.toggleMainMenu());mouseHandler.onClick(".header nav li",(event)=>menuHandler.clickMenuListItem(event));}});})();`,
 }
 
 var JavascriptChecksums = map[string]string{
-	"app": "7d14f00cb219662aaf59f20080265097eb236999dcc712b83775882970d05803",
+	"app": "a70092cda52d5c3673e789868d8cfeb73a890e1a931b102a738021b5c2a65519",
 }

server/static/js/app.js

@@ -324,6 +324,25 @@ class EntryHandler {
         });
         request.execute();
     }
+
+    static fetchOriginalContent(element) {
+        if (element.dataset.completed) {
+            return;
+        }
+
+        element.innerHTML = element.dataset.labelLoading;
+
+        let request = new RequestBuilder(element.dataset.fetchContentUrl);
+        request.withCallback((response) => {
+            element.innerHTML = element.dataset.labelDone;
+            element.dataset.completed = true;
+
+            response.json().then((data) => {
+                document.querySelector(".entry-content").innerHTML = data.content;
+            });
+        });
+        request.execute();
+    }
 }
 
 class ConfirmHandler {
@@ -430,6 +449,15 @@ class NavHandler {
         }
     }
 
+    fetchOriginalContent() {
+        if (! this.isListView()){
+            let link = document.querySelector("a[data-fetch-content-entry]");
+            if (link) {
+                EntryHandler.fetchOriginalContent(link);
+            }
+        }
+    }
+
     toggleEntryStatus() {
         let currentItem = document.querySelector(".current-item");
         if (currentItem !== null) {
@@ -577,6 +605,7 @@ document.addEventListener("DOMContentLoaded", function() {
     keyboardHandler.on("m", () => navHandler.toggleEntryStatus());
     keyboardHandler.on("A", () => navHandler.markPageAsRead());
     keyboardHandler.on("s", () => navHandler.saveEntry());
+    keyboardHandler.on("d", () => navHandler.fetchOriginalContent());
     keyboardHandler.listen();
 
     let mouseHandler = new MouseHandler();
@@ -584,6 +613,12 @@ document.addEventListener("DOMContentLoaded", function() {
         event.preventDefault();
         EntryHandler.saveEntry(event.target);
     });
+
+    mouseHandler.onClick("a[data-fetch-content-entry]", (event) => {
+        event.preventDefault();
+        EntryHandler.fetchOriginalContent(event.target);
+    });
+
     mouseHandler.onClick("a[data-on-click=markPageAsRead]", () => navHandler.markPageAsRead());
     mouseHandler.onClick("a[data-confirm]", (event) => {
         (new ConfirmHandler()).handle(event);

server/template/common.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2017-12-03 17:25:29.427766854 -0800 PST m=+0.040793779
+// 2017-12-10 18:56:24.386027486 -0800 PST m=+0.028006573
 
 package template
 

server/template/html/entry.html

@@ -7,13 +7,26 @@
             <a href="{{ .entry.URL }}" target="_blank" rel="noopener noreferrer" referrerpolicy="no-referrer">{{ .entry.Title }}</a>
         </h1>
         <div class="entry-actions">
-            <a href="#"
+            <ul>
-                title="{{ t "Save this article" }}"
+                <li>
-                data-save-entry="true"
+                    <a href="#"
-                data-save-url="{{ route "saveEntry" "entryID" .entry.ID }}"
+                        title="{{ t "Save this article" }}"
-                data-label-loading="{{ t "Saving..." }}"
+                        data-save-entry="true"
-                data-label-done="{{ t "Done!" }}"
+                        data-save-url="{{ route "saveEntry" "entryID" .entry.ID }}"
-                >{{ t "Save" }}</a>
+                        data-label-loading="{{ t "Saving..." }}"
+                        data-label-done="{{ t "Done!" }}"
+                        >{{ t "Save" }}</a>
+                </li>
+                <li>
+                    <a href="#"
+                        title="{{ t "Fetch original content" }}"
+                        data-fetch-content-entry="true"
+                        data-fetch-content-url="{{ route "fetchContent" "entryID" .entry.ID }}"
+                        data-label-loading="{{ t "Loading..." }}"
+                        data-label-done="{{ t "Done!" }}"
+                        >{{ t "Fetch original content" }}</a>
+                </li>
+            </ul>
         </div>
         <div class="entry-meta">
             <span class="entry-website">

server/template/views.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2017-12-04 20:56:07.05263963 -0800 PST m=+0.018799946
+// 2017-12-10 18:56:24.375327888 -0800 PST m=+0.017306975
 
 package template
 
@@ -466,13 +466,26 @@ var templateViewsMap = map[string]string{
             <a href="{{ .entry.URL }}" target="_blank" rel="noopener noreferrer" referrerpolicy="no-referrer">{{ .entry.Title }}</a>
         </h1>
         <div class="entry-actions">
-            <a href="#"
+            <ul>
-                title="{{ t "Save this article" }}"
+                <li>
-                data-save-entry="true"
+                    <a href="#"
-                data-save-url="{{ route "saveEntry" "entryID" .entry.ID }}"
+                        title="{{ t "Save this article" }}"
-                data-label-loading="{{ t "Saving..." }}"
+                        data-save-entry="true"
-                data-label-done="{{ t "Done!" }}"
+                        data-save-url="{{ route "saveEntry" "entryID" .entry.ID }}"
-                >{{ t "Save" }}</a>
+                        data-label-loading="{{ t "Saving..." }}"
+                        data-label-done="{{ t "Done!" }}"
+                        >{{ t "Save" }}</a>
+                </li>
+                <li>
+                    <a href="#"
+                        title="{{ t "Fetch original content" }}"
+                        data-fetch-content-entry="true"
+                        data-fetch-content-url="{{ route "fetchContent" "entryID" .entry.ID }}"
+                        data-label-loading="{{ t "Loading..." }}"
+                        data-label-done="{{ t "Done!" }}"
+                        >{{ t "Fetch original content" }}</a>
+                </li>
+            </ul>
         </div>
         <div class="entry-meta">
             <span class="entry-website">
@@ -1170,7 +1183,7 @@ var templateViewsMapChecksums = map[string]string{
 	"edit_category":       "cee720faadcec58289b707ad30af623d2ee66c1ce23a732965463250d7ff41c5",
 	"edit_feed":           "c5bc4c22bf7e8348d880395250545595d21fb8c8e723fc5d7cca68e25d250884",
 	"edit_user":           "82d9749d76ddbd2352816d813c4b1f6d92f2222de678b4afe5821090246735c7",
-	"entry":               "7b234e551a98233d9797948db8a000e3d10334e17d5b1d5d17552d1406555b34",
+	"entry":               "ebcf9bb35812dd02759718f7f7411267e6a6c8efd59a9aa0a0e735bcb88efeff",
 	"feed_entries":        "547c19eb36b20e350ce70ed045173b064cdcd6b114afb241c9f2dda9d88fcc27",
 	"feeds":               "c22af39b42ba9ca69ea0914ca789303ec2c5b484abcd4eaa49016e365381257c",
 	"history":             "9a67599a5d8d67ef958e3f07da339b749f42892667547c9e60a54477e8d32a56",

server/ui/controller/entry.go

@@ -8,12 +8,91 @@ import (
 	"errors"
 	"log"
 
+	"github.com/miniflux/miniflux2/integration"
 	"github.com/miniflux/miniflux2/model"
+	"github.com/miniflux/miniflux2/reader/scraper"
 	"github.com/miniflux/miniflux2/server/core"
 	"github.com/miniflux/miniflux2/server/ui/payload"
 	"github.com/miniflux/miniflux2/storage"
 )
 
+// FetchContent downloads the original HTML page and returns relevant contents.
+func (c *Controller) FetchContent(ctx *core.Context, request *core.Request, response *core.Response) {
+	entryID, err := request.IntegerParam("entryID")
+	if err != nil {
+		response.HTML().BadRequest(err)
+		return
+	}
+
+	user := ctx.LoggedUser()
+	builder := c.store.GetEntryQueryBuilder(user.ID, user.Timezone)
+	builder.WithEntryID(entryID)
+	builder.WithoutStatus(model.EntryStatusRemoved)
+
+	entry, err := builder.GetEntry()
+	if err != nil {
+		response.JSON().ServerError(err)
+		return
+	}
+
+	if entry == nil {
+		response.JSON().NotFound(errors.New("Entry not found"))
+		return
+	}
+
+	content, err := scraper.Fetch(entry.URL)
+	if err != nil {
+		response.JSON().ServerError(err)
+		return
+	}
+
+	if len(content) > len(entry.Content) {
+		entry.Content = content
+		c.store.UpdateEntryContent(entry)
+	} else {
+		content = entry.Content
+	}
+
+	response.JSON().Created(map[string]string{"content": content})
+}
+
+// SaveEntry send the link to external services.
+func (c *Controller) SaveEntry(ctx *core.Context, request *core.Request, response *core.Response) {
+	entryID, err := request.IntegerParam("entryID")
+	if err != nil {
+		response.HTML().BadRequest(err)
+		return
+	}
+
+	user := ctx.LoggedUser()
+	builder := c.store.GetEntryQueryBuilder(user.ID, user.Timezone)
+	builder.WithEntryID(entryID)
+	builder.WithoutStatus(model.EntryStatusRemoved)
+
+	entry, err := builder.GetEntry()
+	if err != nil {
+		response.JSON().ServerError(err)
+		return
+	}
+
+	if entry == nil {
+		response.JSON().NotFound(errors.New("Entry not found"))
+		return
+	}
+
+	settings, err := c.store.Integration(user.ID)
+	if err != nil {
+		response.JSON().ServerError(err)
+		return
+	}
+
+	go func() {
+		integration.SendEntry(entry, settings)
+	}()
+
+	response.JSON().Created(map[string]string{"message": "saved"})
+}
+
 // ShowFeedEntry shows a single feed entry in "feed" mode.
 func (c *Controller) ShowFeedEntry(ctx *core.Context, request *core.Request, response *core.Response) {
 	user := ctx.LoggedUser()

server/ui/controller/integrations.go

@@ -6,11 +6,8 @@ package controller
 
 import (
 	"crypto/md5"
-	"errors"
 	"fmt"
 
-	"github.com/miniflux/miniflux2/integration"
-	"github.com/miniflux/miniflux2/model"
 	"github.com/miniflux/miniflux2/server/core"
 	"github.com/miniflux/miniflux2/server/ui/form"
 )
@@ -73,40 +70,3 @@ func (c *Controller) UpdateIntegration(ctx *core.Context, request *core.Request,
 
 	response.Redirect(ctx.Route("integrations"))
 }
-
-// SaveEntry send the link to external services.
-func (c *Controller) SaveEntry(ctx *core.Context, request *core.Request, response *core.Response) {
-	entryID, err := request.IntegerParam("entryID")
-	if err != nil {
-		response.HTML().BadRequest(err)
-		return
-	}
-
-	user := ctx.LoggedUser()
-	builder := c.store.GetEntryQueryBuilder(user.ID, user.Timezone)
-	builder.WithEntryID(entryID)
-	builder.WithoutStatus(model.EntryStatusRemoved)
-
-	entry, err := builder.GetEntry()
-	if err != nil {
-		response.JSON().ServerError(err)
-		return
-	}
-
-	if entry == nil {
-		response.JSON().NotFound(errors.New("Entry not found"))
-		return
-	}
-
-	settings, err := c.store.Integration(user.ID)
-	if err != nil {
-		response.JSON().ServerError(err)
-		return
-	}
-
-	go func() {
-		integration.SendEntry(entry, settings)
-	}()
-
-	response.JSON().Created(map[string]string{"message": "saved"})
-}

sql/sql.go

@@ -1,5 +1,5 @@
 // Code generated by go generate; DO NOT EDIT.
-// 2017-12-03 17:25:29.391052668 -0800 PST m=+0.004079593
+// 2017-12-10 18:56:24.36359961 -0800 PST m=+0.005578697
 
 package sql
 

storage/entry.go

@@ -59,6 +59,23 @@ func (s *Storage) CreateEntry(entry *model.Entry) error {
 	return nil
 }
 
+// UpdateEntryContent updates entry content.
+func (s *Storage) UpdateEntryContent(entry *model.Entry) error {
+	query := `
+		UPDATE entries SET
+		content=$1
+		WHERE user_id=$2 AND id=$3
+	`
+
+	_, err := s.db.Exec(
+		query,
+		entry.Content,
+		entry.UserID,
+		entry.ID,
+	)
+	return err
+}
+
 // UpdateEntry update an entry when a feed is refreshed.
 func (s *Storage) UpdateEntry(entry *model.Entry) error {
 	query := `