1
0
Fork 0
miniflux/internal/reader/icon/finder.go

215 lines
5.4 KiB
Go
Raw Normal View History

// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
2017-11-20 00:10:04 -05:00
package icon // import "miniflux.app/v2/internal/reader/icon"
2017-11-20 00:10:04 -05:00
import (
2017-12-22 22:01:39 -05:00
"encoding/base64"
2017-11-20 00:10:04 -05:00
"fmt"
"io"
"net/url"
2017-12-22 22:01:39 -05:00
"strings"
"miniflux.app/v2/internal/config"
"miniflux.app/v2/internal/crypto"
"miniflux.app/v2/internal/http/client"
"miniflux.app/v2/internal/logger"
"miniflux.app/v2/internal/model"
"miniflux.app/v2/internal/urllib"
2017-11-20 00:10:04 -05:00
"github.com/PuerkitoBio/goquery"
)
// FindIcon try to find the website's icon.
func FindIcon(websiteURL, feedIconURL, userAgent string, fetchViaProxy, allowSelfSignedCertificates bool) (icon *model.Icon, err error) {
if feedIconURL == "" {
feedIconURL, err = fetchHTMLDocumentAndFindIconURL(websiteURL, userAgent, fetchViaProxy, allowSelfSignedCertificates)
if err != nil {
return nil, err
}
}
if strings.HasPrefix(feedIconURL, "data:") {
return parseImageDataURL(feedIconURL)
}
feedIconURL, err = generateIconURL(websiteURL, feedIconURL)
if err != nil {
return nil, err
}
if icon, err = downloadIcon(feedIconURL, userAgent, fetchViaProxy, allowSelfSignedCertificates); err != nil {
return nil, err
}
2017-11-20 00:10:04 -05:00
return icon, nil
}
func generateIconURL(websiteURL, feedIconURL string) (iconURL string, err error) {
feedIconURL = strings.TrimSpace(feedIconURL)
2017-11-20 00:10:04 -05:00
if feedIconURL == "" {
iconURL, err = urllib.JoinBaseURLAndPath(urllib.RootURL(websiteURL), "favicon.ico")
if err != nil {
return "", fmt.Errorf(`icon: unable to join base URL and path: %w`, err)
}
} else {
iconURL, err = urllib.AbsoluteURL(websiteURL, feedIconURL)
if err != nil {
return "", fmt.Errorf(`icon: unable to convert icon URL to absolute URL: %w`, err)
}
2017-11-20 00:10:04 -05:00
}
return iconURL, nil
}
func fetchHTMLDocumentAndFindIconURL(websiteURL, userAgent string, fetchViaProxy, allowSelfSignedCertificates bool) (string, error) {
rootURL := urllib.RootURL(websiteURL)
logger.Debug("[FindIcon] Find icon from HTML webpage: %s", rootURL)
clt := client.NewClientWithConfig(rootURL, config.Opts)
clt.WithUserAgent(userAgent)
clt.AllowSelfSignedCertificates = allowSelfSignedCertificates
if fetchViaProxy {
clt.WithProxy()
2017-12-22 22:01:39 -05:00
}
response, err := clt.Get()
2017-11-20 00:10:04 -05:00
if err != nil {
return "", fmt.Errorf("icon: unable to download website index page: %v", err)
2017-11-20 00:10:04 -05:00
}
if response.HasServerFailure() {
return "", fmt.Errorf("icon: unable to download website index page: status=%d", response.StatusCode)
}
return findIconURLFromHTMLDocument(response.Body)
2017-11-20 00:10:04 -05:00
}
func findIconURLFromHTMLDocument(body io.Reader) (string, error) {
2017-11-20 00:10:04 -05:00
queries := []string{
"link[rel='shortcut icon']",
"link[rel='Shortcut Icon']",
"link[rel='icon shortcut']",
"link[rel='icon']",
}
doc, err := goquery.NewDocumentFromReader(body)
2017-11-20 00:10:04 -05:00
if err != nil {
return "", fmt.Errorf("icon: unable to read document: %v", err)
2017-11-20 00:10:04 -05:00
}
var iconURL string
for _, query := range queries {
doc.Find(query).Each(func(i int, s *goquery.Selection) {
if href, exists := s.Attr("href"); exists {
2020-11-06 20:11:52 -05:00
iconURL = strings.TrimSpace(href)
2017-11-20 00:10:04 -05:00
}
})
if iconURL != "" {
break
}
}
return iconURL, nil
}
func downloadIcon(iconURL, userAgent string, fetchViaProxy, allowSelfSignedCertificates bool) (*model.Icon, error) {
clt := client.NewClientWithConfig(iconURL, config.Opts)
clt.WithUserAgent(userAgent)
clt.AllowSelfSignedCertificates = allowSelfSignedCertificates
if fetchViaProxy {
clt.WithProxy()
}
2018-04-28 13:51:07 -04:00
response, err := clt.Get()
2017-11-20 00:10:04 -05:00
if err != nil {
return nil, fmt.Errorf("icon: unable to download iconURL: %v", err)
2017-11-20 00:10:04 -05:00
}
if response.HasServerFailure() {
return nil, fmt.Errorf("icon: unable to download icon: status=%d", response.StatusCode)
2017-11-20 00:10:04 -05:00
}
body, err := io.ReadAll(response.Body)
2017-11-20 00:10:04 -05:00
if err != nil {
return nil, fmt.Errorf("icon: unable to read downloaded icon: %v", err)
2017-11-20 00:10:04 -05:00
}
if len(body) == 0 {
return nil, fmt.Errorf("icon: downloaded icon is empty, iconURL=%s", iconURL)
2017-11-20 00:10:04 -05:00
}
icon := &model.Icon{
2018-01-02 22:15:08 -05:00
Hash: crypto.HashFromBytes(body),
2017-11-20 00:10:04 -05:00
MimeType: response.ContentType,
Content: body,
}
return icon, nil
}
2017-12-22 22:01:39 -05:00
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs#syntax
// data:[<mediatype>][;base64],<data>
2017-12-22 22:01:39 -05:00
func parseImageDataURL(value string) (*model.Icon, error) {
var mediaType string
var encoding string
if !strings.HasPrefix(value, "data:") {
return nil, fmt.Errorf(`icon: invalid data URL (missing data:) %q`, value)
}
value = value[5:]
2017-12-22 22:01:39 -05:00
comma := strings.Index(value, ",")
if comma < 0 {
return nil, fmt.Errorf(`icon: invalid data URL (no comma) %q`, value)
2017-12-22 22:01:39 -05:00
}
data := value[comma+1:]
semicolon := strings.Index(value[0:comma], ";")
2017-12-22 22:01:39 -05:00
if semicolon > 0 {
mediaType = value[0:semicolon]
encoding = value[semicolon+1 : comma]
} else {
mediaType = value[0:comma]
2017-12-22 22:01:39 -05:00
}
if !strings.HasPrefix(mediaType, "image/") {
return nil, fmt.Errorf(`icon: invalid media type %q`, mediaType)
2017-12-22 22:01:39 -05:00
}
var blob []byte
switch encoding {
case "base64":
var err error
blob, err = base64.StdEncoding.DecodeString(data)
if err != nil {
return nil, fmt.Errorf(`icon: invalid data %q (%v)`, value, err)
}
case "":
decodedData, err := url.QueryUnescape(data)
if err != nil {
return nil, fmt.Errorf(`icon: unable to decode data URL %q`, value)
}
blob = []byte(decodedData)
default:
return nil, fmt.Errorf(`icon: unsupported data URL encoding %q`, value)
2017-12-22 22:01:39 -05:00
}
if len(blob) == 0 {
return nil, fmt.Errorf(`icon: empty data URL %q`, value)
2017-12-22 22:01:39 -05:00
}
icon := &model.Icon{
2018-01-02 22:15:08 -05:00
Hash: crypto.HashFromBytes(blob),
2017-12-22 22:01:39 -05:00
Content: blob,
MimeType: mediaType,
2017-12-22 22:01:39 -05:00
}
return icon, nil
}