2023-06-19 17:42:47 -04:00
|
|
|
// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
|
|
|
|
// SPDX-License-Identifier: Apache-2.0
|
2017-11-20 00:10:04 -05:00
|
|
|
|
2023-08-10 22:46:45 -04:00
|
|
|
package icon // import "miniflux.app/v2/internal/reader/icon"
|
2017-11-20 00:10:04 -05:00
|
|
|
|
|
|
|
import (
|
2017-12-22 22:01:39 -05:00
|
|
|
"encoding/base64"
|
2017-11-20 00:10:04 -05:00
|
|
|
"fmt"
|
2017-11-20 20:12:37 -05:00
|
|
|
"io"
|
2023-08-13 22:09:01 -04:00
|
|
|
"net/url"
|
2017-12-22 22:01:39 -05:00
|
|
|
"strings"
|
2017-11-20 20:12:37 -05:00
|
|
|
|
2023-08-10 22:46:45 -04:00
|
|
|
"miniflux.app/v2/internal/config"
|
|
|
|
"miniflux.app/v2/internal/crypto"
|
|
|
|
"miniflux.app/v2/internal/http/client"
|
|
|
|
"miniflux.app/v2/internal/logger"
|
|
|
|
"miniflux.app/v2/internal/model"
|
2023-08-13 22:09:01 -04:00
|
|
|
"miniflux.app/v2/internal/urllib"
|
2017-11-20 00:10:04 -05:00
|
|
|
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
|
|
)
|
|
|
|
|
|
|
|
// FindIcon try to find the website's icon.
|
2023-09-09 17:18:39 -04:00
|
|
|
func FindIcon(websiteURL, feedIconURL, userAgent string, fetchViaProxy, allowSelfSignedCertificates bool) (icon *model.Icon, err error) {
|
|
|
|
if feedIconURL == "" {
|
|
|
|
feedIconURL, err = fetchHTMLDocumentAndFindIconURL(websiteURL, userAgent, fetchViaProxy, allowSelfSignedCertificates)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
}
|
2022-01-08 18:09:12 -05:00
|
|
|
|
2023-09-09 17:18:39 -04:00
|
|
|
if strings.HasPrefix(feedIconURL, "data:") {
|
|
|
|
return parseImageDataURL(feedIconURL)
|
|
|
|
}
|
2022-01-08 18:09:12 -05:00
|
|
|
|
2023-09-09 17:18:39 -04:00
|
|
|
feedIconURL, err = generateIconURL(websiteURL, feedIconURL)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2021-02-21 16:42:49 -05:00
|
|
|
|
2023-09-09 17:18:39 -04:00
|
|
|
if icon, err = downloadIcon(feedIconURL, userAgent, fetchViaProxy, allowSelfSignedCertificates); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2017-11-20 00:10:04 -05:00
|
|
|
|
2023-09-09 17:18:39 -04:00
|
|
|
return icon, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func generateIconURL(websiteURL, feedIconURL string) (iconURL string, err error) {
|
|
|
|
feedIconURL = strings.TrimSpace(feedIconURL)
|
2017-11-20 00:10:04 -05:00
|
|
|
|
2023-09-09 17:18:39 -04:00
|
|
|
if feedIconURL == "" {
|
|
|
|
iconURL, err = urllib.JoinBaseURLAndPath(urllib.RootURL(websiteURL), "favicon.ico")
|
2023-06-04 18:01:59 -04:00
|
|
|
if err != nil {
|
2023-09-09 17:18:39 -04:00
|
|
|
return "", fmt.Errorf(`icon: unable to join base URL and path: %w`, err)
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
iconURL, err = urllib.AbsoluteURL(websiteURL, feedIconURL)
|
|
|
|
if err != nil {
|
|
|
|
return "", fmt.Errorf(`icon: unable to convert icon URL to absolute URL: %w`, err)
|
2023-06-04 18:01:59 -04:00
|
|
|
}
|
2017-11-20 00:10:04 -05:00
|
|
|
}
|
|
|
|
|
2023-09-09 17:18:39 -04:00
|
|
|
return iconURL, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func fetchHTMLDocumentAndFindIconURL(websiteURL, userAgent string, fetchViaProxy, allowSelfSignedCertificates bool) (string, error) {
|
|
|
|
rootURL := urllib.RootURL(websiteURL)
|
|
|
|
logger.Debug("[FindIcon] Find icon from HTML webpage: %s", rootURL)
|
|
|
|
|
|
|
|
clt := client.NewClientWithConfig(rootURL, config.Opts)
|
|
|
|
clt.WithUserAgent(userAgent)
|
|
|
|
clt.AllowSelfSignedCertificates = allowSelfSignedCertificates
|
|
|
|
|
|
|
|
if fetchViaProxy {
|
|
|
|
clt.WithProxy()
|
2017-12-22 22:01:39 -05:00
|
|
|
}
|
|
|
|
|
2023-09-09 17:18:39 -04:00
|
|
|
response, err := clt.Get()
|
2017-11-20 00:10:04 -05:00
|
|
|
if err != nil {
|
2023-09-09 17:18:39 -04:00
|
|
|
return "", fmt.Errorf("icon: unable to download website index page: %v", err)
|
2017-11-20 00:10:04 -05:00
|
|
|
}
|
|
|
|
|
2023-09-09 17:18:39 -04:00
|
|
|
if response.HasServerFailure() {
|
|
|
|
return "", fmt.Errorf("icon: unable to download website index page: status=%d", response.StatusCode)
|
|
|
|
}
|
|
|
|
|
|
|
|
return findIconURLFromHTMLDocument(response.Body)
|
2017-11-20 00:10:04 -05:00
|
|
|
}
|
|
|
|
|
2023-09-09 17:18:39 -04:00
|
|
|
func findIconURLFromHTMLDocument(body io.Reader) (string, error) {
|
2017-11-20 00:10:04 -05:00
|
|
|
queries := []string{
|
|
|
|
"link[rel='shortcut icon']",
|
|
|
|
"link[rel='Shortcut Icon']",
|
|
|
|
"link[rel='icon shortcut']",
|
|
|
|
"link[rel='icon']",
|
|
|
|
}
|
|
|
|
|
2023-09-09 17:18:39 -04:00
|
|
|
doc, err := goquery.NewDocumentFromReader(body)
|
2017-11-20 00:10:04 -05:00
|
|
|
if err != nil {
|
2022-01-08 18:09:12 -05:00
|
|
|
return "", fmt.Errorf("icon: unable to read document: %v", err)
|
2017-11-20 00:10:04 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
var iconURL string
|
|
|
|
for _, query := range queries {
|
|
|
|
doc.Find(query).Each(func(i int, s *goquery.Selection) {
|
|
|
|
if href, exists := s.Attr("href"); exists {
|
2020-11-06 20:11:52 -05:00
|
|
|
iconURL = strings.TrimSpace(href)
|
2017-11-20 00:10:04 -05:00
|
|
|
}
|
|
|
|
})
|
|
|
|
|
|
|
|
if iconURL != "" {
|
|
|
|
break
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return iconURL, nil
|
|
|
|
}
|
|
|
|
|
2022-01-08 18:09:12 -05:00
|
|
|
func downloadIcon(iconURL, userAgent string, fetchViaProxy, allowSelfSignedCertificates bool) (*model.Icon, error) {
|
2020-09-27 17:29:48 -04:00
|
|
|
clt := client.NewClientWithConfig(iconURL, config.Opts)
|
2022-01-08 18:09:12 -05:00
|
|
|
clt.WithUserAgent(userAgent)
|
2021-02-21 16:42:49 -05:00
|
|
|
clt.AllowSelfSignedCertificates = allowSelfSignedCertificates
|
2020-09-10 02:28:54 -04:00
|
|
|
if fetchViaProxy {
|
|
|
|
clt.WithProxy()
|
|
|
|
}
|
2022-01-08 18:09:12 -05:00
|
|
|
|
2018-04-28 13:51:07 -04:00
|
|
|
response, err := clt.Get()
|
2017-11-20 00:10:04 -05:00
|
|
|
if err != nil {
|
2022-01-08 18:09:12 -05:00
|
|
|
return nil, fmt.Errorf("icon: unable to download iconURL: %v", err)
|
2017-11-20 00:10:04 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
if response.HasServerFailure() {
|
2022-01-08 18:09:12 -05:00
|
|
|
return nil, fmt.Errorf("icon: unable to download icon: status=%d", response.StatusCode)
|
2017-11-20 00:10:04 -05:00
|
|
|
}
|
|
|
|
|
2021-02-17 00:19:03 -05:00
|
|
|
body, err := io.ReadAll(response.Body)
|
2017-11-20 00:10:04 -05:00
|
|
|
if err != nil {
|
2022-01-08 18:09:12 -05:00
|
|
|
return nil, fmt.Errorf("icon: unable to read downloaded icon: %v", err)
|
2017-11-20 00:10:04 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
if len(body) == 0 {
|
2022-01-08 18:09:12 -05:00
|
|
|
return nil, fmt.Errorf("icon: downloaded icon is empty, iconURL=%s", iconURL)
|
2017-11-20 00:10:04 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
icon := &model.Icon{
|
2018-01-02 22:15:08 -05:00
|
|
|
Hash: crypto.HashFromBytes(body),
|
2017-11-20 00:10:04 -05:00
|
|
|
MimeType: response.ContentType,
|
|
|
|
Content: body,
|
|
|
|
}
|
|
|
|
|
|
|
|
return icon, nil
|
|
|
|
}
|
2017-12-22 22:01:39 -05:00
|
|
|
|
2022-03-26 01:05:19 -04:00
|
|
|
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs#syntax
|
|
|
|
// data:[<mediatype>][;base64],<data>
|
2017-12-22 22:01:39 -05:00
|
|
|
func parseImageDataURL(value string) (*model.Icon, error) {
|
2022-03-26 01:05:19 -04:00
|
|
|
var mediaType string
|
|
|
|
var encoding string
|
|
|
|
|
|
|
|
if !strings.HasPrefix(value, "data:") {
|
|
|
|
return nil, fmt.Errorf(`icon: invalid data URL (missing data:) %q`, value)
|
|
|
|
}
|
|
|
|
|
|
|
|
value = value[5:]
|
2017-12-22 22:01:39 -05:00
|
|
|
|
2022-03-26 01:05:19 -04:00
|
|
|
comma := strings.Index(value, ",")
|
|
|
|
if comma < 0 {
|
|
|
|
return nil, fmt.Errorf(`icon: invalid data URL (no comma) %q`, value)
|
2017-12-22 22:01:39 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
data := value[comma+1:]
|
2022-03-26 01:05:19 -04:00
|
|
|
semicolon := strings.Index(value[0:comma], ";")
|
2017-12-22 22:01:39 -05:00
|
|
|
|
2022-03-26 01:05:19 -04:00
|
|
|
if semicolon > 0 {
|
|
|
|
mediaType = value[0:semicolon]
|
|
|
|
encoding = value[semicolon+1 : comma]
|
|
|
|
} else {
|
|
|
|
mediaType = value[0:comma]
|
2017-12-22 22:01:39 -05:00
|
|
|
}
|
|
|
|
|
2022-03-26 01:05:19 -04:00
|
|
|
if !strings.HasPrefix(mediaType, "image/") {
|
|
|
|
return nil, fmt.Errorf(`icon: invalid media type %q`, mediaType)
|
2017-12-22 22:01:39 -05:00
|
|
|
}
|
|
|
|
|
2022-03-26 01:05:19 -04:00
|
|
|
var blob []byte
|
|
|
|
switch encoding {
|
|
|
|
case "base64":
|
|
|
|
var err error
|
|
|
|
blob, err = base64.StdEncoding.DecodeString(data)
|
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf(`icon: invalid data %q (%v)`, value, err)
|
|
|
|
}
|
|
|
|
case "":
|
2023-08-13 22:09:01 -04:00
|
|
|
decodedData, err := url.QueryUnescape(data)
|
2022-03-26 01:05:19 -04:00
|
|
|
if err != nil {
|
|
|
|
return nil, fmt.Errorf(`icon: unable to decode data URL %q`, value)
|
|
|
|
}
|
|
|
|
blob = []byte(decodedData)
|
|
|
|
default:
|
|
|
|
return nil, fmt.Errorf(`icon: unsupported data URL encoding %q`, value)
|
2017-12-22 22:01:39 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
if len(blob) == 0 {
|
2022-03-26 01:05:19 -04:00
|
|
|
return nil, fmt.Errorf(`icon: empty data URL %q`, value)
|
2017-12-22 22:01:39 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
icon := &model.Icon{
|
2018-01-02 22:15:08 -05:00
|
|
|
Hash: crypto.HashFromBytes(blob),
|
2017-12-22 22:01:39 -05:00
|
|
|
Content: blob,
|
2022-03-26 01:05:19 -04:00
|
|
|
MimeType: mediaType,
|
2017-12-22 22:01:39 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
return icon, nil
|
|
|
|
}
|