miniflux/internal/reader/icon/finder.go

// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package icon // import "miniflux.app/v2/internal/reader/icon"

import (
	"encoding/base64"
	"fmt"
	"io"
	"log/slog"
	"net/url"
	"strings"

	"miniflux.app/v2/internal/config"
	"miniflux.app/v2/internal/crypto"
	"miniflux.app/v2/internal/http/client"
	"miniflux.app/v2/internal/model"
	"miniflux.app/v2/internal/urllib"

	"github.com/PuerkitoBio/goquery"
)

type IconFinder struct {
	websiteURL                  string
	feedIconURL                 string
	userAgent                   string
	fetchViaProxy               bool
	allowSelfSignedCertificates bool
}

func NewIconFinder(websiteURL, feedIconURL, userAgent string, fetchViaProxy, allowSelfSignedCertificates bool) *IconFinder {
	return &IconFinder{
		websiteURL:                  websiteURL,
		feedIconURL:                 feedIconURL,
		userAgent:                   userAgent,
		fetchViaProxy:               fetchViaProxy,
		allowSelfSignedCertificates: allowSelfSignedCertificates,
	}
}

func (f *IconFinder) FindIcon() (*model.Icon, error) {
	slog.Debug("Begin icon discovery process",
		slog.String("website_url", f.websiteURL),
		slog.String("feed_icon_url", f.feedIconURL),
	)

	if f.feedIconURL != "" {
		if icon, err := f.FetchFeedIcon(); err != nil {
			slog.Debug("Unable to download icon from feed",
				slog.String("website_url", f.websiteURL),
				slog.String("feed_icon_url", f.feedIconURL),
				slog.Any("error", err),
			)
		} else if icon != nil {
			return icon, nil
		}
	}

	if icon, err := f.FetchIconsFromHTMLDocument(); err != nil {
		slog.Debug("Unable to fetch icons from HTML document",
			slog.String("website_url", f.websiteURL),
			slog.Any("error", err),
		)
	} else if icon != nil {
		return icon, nil
	}

	return f.FetchDefaultIcon()
}

func (f *IconFinder) FetchDefaultIcon() (*model.Icon, error) {
	slog.Debug("Fetching default icon",
		slog.String("website_url", f.websiteURL),
	)

	iconURL, err := urllib.JoinBaseURLAndPath(urllib.RootURL(f.websiteURL), "favicon.ico")
	if err != nil {
		return nil, fmt.Errorf(`icon: unable to join root URL and path: %w`, err)
	}

	icon, err := f.DownloadIcon(iconURL)
	if err != nil {
		return nil, err
	}

	return icon, nil
}

func (f *IconFinder) FetchFeedIcon() (*model.Icon, error) {
	slog.Debug("Fetching feed icon",
		slog.String("website_url", f.websiteURL),
		slog.String("feed_icon_url", f.feedIconURL),
	)

	iconURL, err := urllib.AbsoluteURL(f.websiteURL, f.feedIconURL)
	if err != nil {
		return nil, fmt.Errorf(`icon: unable to convert icon URL to absolute URL: %w`, err)
	}

	return f.DownloadIcon(iconURL)
}

func (f *IconFinder) FetchIconsFromHTMLDocument() (*model.Icon, error) {
	slog.Debug("Searching icons from HTML document",
		slog.String("website_url", f.websiteURL),
	)

	documentBody, err := f.FetchRootDocument()
	if err != nil {
		return nil, err
	}

	iconURLs, err := findIconURLsFromHTMLDocument(documentBody)
	if err != nil {
		return nil, err
	}

	slog.Debug("Searched icon from HTML document",
		slog.String("website_url", f.websiteURL),
		slog.String("icon_urls", strings.Join(iconURLs, ",")),
	)

	for _, iconURL := range iconURLs {
		if strings.HasPrefix(iconURL, "data:") {
			slog.Debug("Found icon with data URL",
				slog.String("website_url", f.websiteURL),
			)
			return parseImageDataURL(iconURL)
		}

		iconURL, err = urllib.AbsoluteURL(f.websiteURL, iconURL)
		if err != nil {
			return nil, fmt.Errorf(`icon: unable to convert icon URL to absolute URL: %w`, err)
		}

		if icon, err := f.DownloadIcon(iconURL); err != nil {
			slog.Debug("Unable to download icon from HTML document",
				slog.String("website_url", f.websiteURL),
				slog.String("icon_url", iconURL),
				slog.Any("error", err),
			)
		} else if icon != nil {
			slog.Debug("Found icon from HTML document",
				slog.String("website_url", f.websiteURL),
				slog.String("icon_url", iconURL),
			)
			return icon, nil
		}
	}

	return nil, nil
}

func (f *IconFinder) FetchRootDocument() (io.Reader, error) {
	rootURL := urllib.RootURL(f.websiteURL)

	clt := client.NewClientWithConfig(rootURL, config.Opts)
	clt.WithUserAgent(f.userAgent)
	clt.AllowSelfSignedCertificates = f.allowSelfSignedCertificates

	if f.fetchViaProxy {
		clt.WithProxy()
	}

	response, err := clt.Get()
	if err != nil {
		return nil, fmt.Errorf("icon: unable to download website index page: %v", err)
	}

	if response.HasServerFailure() {
		return nil, fmt.Errorf("icon: unable to download website index page: status=%d", response.StatusCode)
	}

	return response.Body, nil
}

func (f *IconFinder) DownloadIcon(iconURL string) (*model.Icon, error) {
	slog.Debug("Downloading icon",
		slog.String("website_url", f.websiteURL),
		slog.String("icon_url", iconURL),
	)

	clt := client.NewClientWithConfig(iconURL, config.Opts)
	clt.WithUserAgent(f.userAgent)
	clt.AllowSelfSignedCertificates = f.allowSelfSignedCertificates
	if f.fetchViaProxy {
		clt.WithProxy()
	}

	response, err := clt.Get()
	if err != nil {
		return nil, fmt.Errorf("icon: unable to download iconURL: %v", err)
	}

	if response.HasServerFailure() {
		return nil, fmt.Errorf("icon: unable to download icon: status=%d", response.StatusCode)
	}

	body, err := io.ReadAll(response.Body)
	if err != nil {
		return nil, fmt.Errorf("icon: unable to read downloaded icon: %v", err)
	}

	if len(body) == 0 {
		return nil, fmt.Errorf("icon: downloaded icon is empty, iconURL=%s", iconURL)
	}

	icon := &model.Icon{
		Hash:     crypto.HashFromBytes(body),
		MimeType: response.ContentType,
		Content:  body,
	}

	return icon, nil
}

func findIconURLsFromHTMLDocument(body io.Reader) ([]string, error) {
	queries := []string{
		"link[rel='shortcut icon']",
		"link[rel='Shortcut Icon']",
		"link[rel='icon shortcut']",
		"link[rel='icon']",
	}

	doc, err := goquery.NewDocumentFromReader(body)
	if err != nil {
		return nil, fmt.Errorf("icon: unable to read document: %v", err)
	}

	var iconURLs []string
	for _, query := range queries {
		slog.Debug("Searching icon URL in HTML document", slog.String("query", query))

		doc.Find(query).Each(func(i int, s *goquery.Selection) {
			var iconURL string

			if href, exists := s.Attr("href"); exists {
				iconURL = strings.TrimSpace(href)
			}

			if iconURL != "" {
				iconURLs = append(iconURLs, iconURL)

				slog.Debug("Found icon URL in HTML document",
					slog.String("query", query),
					slog.String("icon_url", iconURL))
			}
		})
	}

	return iconURLs, nil
}

// https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs#syntax
// data:[<mediatype>][;base64],<data>
func parseImageDataURL(value string) (*model.Icon, error) {
	var mediaType string
	var encoding string

	if !strings.HasPrefix(value, "data:") {
		return nil, fmt.Errorf(`icon: invalid data URL (missing data:) %q`, value)
	}

	value = value[5:]

	comma := strings.Index(value, ",")
	if comma < 0 {
		return nil, fmt.Errorf(`icon: invalid data URL (no comma) %q`, value)
	}

	data := value[comma+1:]
	semicolon := strings.Index(value[0:comma], ";")

	if semicolon > 0 {
		mediaType = value[0:semicolon]
		encoding = value[semicolon+1 : comma]
	} else {
		mediaType = value[0:comma]
	}

	if !strings.HasPrefix(mediaType, "image/") {
		return nil, fmt.Errorf(`icon: invalid media type %q`, mediaType)
	}

	var blob []byte
	switch encoding {
	case "base64":
		var err error
		blob, err = base64.StdEncoding.DecodeString(data)
		if err != nil {
			return nil, fmt.Errorf(`icon: invalid data %q (%v)`, value, err)
		}
	case "":
		decodedData, err := url.QueryUnescape(data)
		if err != nil {
			return nil, fmt.Errorf(`icon: unable to decode data URL %q`, value)
		}
		blob = []byte(decodedData)
	case "utf8":
		blob = []byte(data)
	default:
		return nil, fmt.Errorf(`icon: unsupported data URL encoding %q`, value)
	}

	if len(blob) == 0 {
		return nil, fmt.Errorf(`icon: empty data URL %q`, value)
	}

	icon := &model.Icon{
		Hash:     crypto.HashFromBytes(blob),
		Content:  blob,
		MimeType: mediaType,
	}

	return icon, nil
}