diff --git a/reader/handler/handler.go b/reader/handler/handler.go index 49f0a5b7..13f851d5 100644 --- a/reader/handler/handler.go +++ b/reader/handler/handler.go @@ -90,6 +90,7 @@ func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model store, subscription.ID, subscription.SiteURL, + feedCreationRequest.UserAgent, feedCreationRequest.FetchViaProxy, feedCreationRequest.AllowSelfSignedCertificates, ) @@ -178,6 +179,7 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64) error { store, originalFeed.ID, originalFeed.SiteURL, + originalFeed.UserAgent, originalFeed.FetchViaProxy, originalFeed.AllowSelfSignedCertificates, ) @@ -196,9 +198,9 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64) error { return nil } -func checkFeedIcon(store *storage.Storage, feedID int64, websiteURL string, fetchViaProxy, allowSelfSignedCertificates bool) { +func checkFeedIcon(store *storage.Storage, feedID int64, websiteURL, userAgent string, fetchViaProxy, allowSelfSignedCertificates bool) { if !store.HasIcon(feedID) { - icon, err := icon.FindIcon(websiteURL, fetchViaProxy, allowSelfSignedCertificates) + icon, err := icon.FindIcon(websiteURL, userAgent, fetchViaProxy, allowSelfSignedCertificates) if err != nil { logger.Debug(`[CheckFeedIcon] %v (feedID=%d websiteURL=%s)`, err, feedID, websiteURL) } else if icon == nil { diff --git a/reader/icon/finder.go b/reader/icon/finder.go index e53fcd6e..e06f6d49 100644 --- a/reader/icon/finder.go +++ b/reader/icon/finder.go @@ -21,21 +21,25 @@ import ( ) // FindIcon try to find the website's icon. -func FindIcon(websiteURL string, fetchViaProxy, allowSelfSignedCertificates bool) (*model.Icon, error) { +func FindIcon(websiteURL, userAgent string, fetchViaProxy, allowSelfSignedCertificates bool) (*model.Icon, error) { rootURL := url.RootURL(websiteURL) + logger.Debug("[FindIcon] Trying to find an icon: rootURL=%q websiteURL=%q userAgent=%q", rootURL, websiteURL, userAgent) + clt := client.NewClientWithConfig(rootURL, config.Opts) + clt.WithUserAgent(userAgent) clt.AllowSelfSignedCertificates = allowSelfSignedCertificates + if fetchViaProxy { clt.WithProxy() } response, err := clt.Get() if err != nil { - return nil, fmt.Errorf("unable to download website index page: %v", err) + return nil, fmt.Errorf("icon: unable to download website index page: %v", err) } if response.HasServerFailure() { - return nil, fmt.Errorf("unable to download website index page: status=%d", response.StatusCode) + return nil, fmt.Errorf("icon: unable to download website index page: status=%d", response.StatusCode) } iconURL, err := parseDocument(rootURL, response.Body) @@ -48,7 +52,7 @@ func FindIcon(websiteURL string, fetchViaProxy, allowSelfSignedCertificates bool } logger.Debug("[FindIcon] Fetching icon => %s", iconURL) - icon, err := downloadIcon(iconURL, fetchViaProxy, allowSelfSignedCertificates) + icon, err := downloadIcon(iconURL, userAgent, fetchViaProxy, allowSelfSignedCertificates) if err != nil { return nil, err } @@ -66,7 +70,7 @@ func parseDocument(websiteURL string, data io.Reader) (string, error) { doc, err := goquery.NewDocumentFromReader(data) if err != nil { - return "", fmt.Errorf("unable to read document: %v", err) + return "", fmt.Errorf("icon: unable to read document: %v", err) } var iconURL string @@ -91,28 +95,30 @@ func parseDocument(websiteURL string, data io.Reader) (string, error) { return iconURL, nil } -func downloadIcon(iconURL string, fetchViaProxy, allowSelfSignedCertificates bool) (*model.Icon, error) { +func downloadIcon(iconURL, userAgent string, fetchViaProxy, allowSelfSignedCertificates bool) (*model.Icon, error) { clt := client.NewClientWithConfig(iconURL, config.Opts) + clt.WithUserAgent(userAgent) clt.AllowSelfSignedCertificates = allowSelfSignedCertificates if fetchViaProxy { clt.WithProxy() } + response, err := clt.Get() if err != nil { - return nil, fmt.Errorf("unable to download iconURL: %v", err) + return nil, fmt.Errorf("icon: unable to download iconURL: %v", err) } if response.HasServerFailure() { - return nil, fmt.Errorf("unable to download icon: status=%d", response.StatusCode) + return nil, fmt.Errorf("icon: unable to download icon: status=%d", response.StatusCode) } body, err := io.ReadAll(response.Body) if err != nil { - return nil, fmt.Errorf("unable to read downloaded icon: %v", err) + return nil, fmt.Errorf("icon: unable to read downloaded icon: %v", err) } if len(body) == 0 { - return nil, fmt.Errorf("downloaded icon is empty, iconURL=%s", iconURL) + return nil, fmt.Errorf("icon: downloaded icon is empty, iconURL=%s", iconURL) } icon := &model.Icon{ diff --git a/ui/proxy.go b/ui/proxy.go index 0f7630ce..663eea45 100644 --- a/ui/proxy.go +++ b/ui/proxy.go @@ -60,6 +60,7 @@ func (h *handler) imageProxy(w http.ResponseWriter, r *http.Request) { defer resp.Body.Close() if resp.StatusCode != http.StatusOK { + logger.Error(`[Proxy] code=%d url=%q`, resp.StatusCode, imageURL) html.NotFound(w, r) return }