Use custom feed user agent to fetch website icon
This commit is contained in:
parent
8329e9b46c
commit
2309b27458
3 changed files with 21 additions and 12 deletions
|
@ -90,6 +90,7 @@ func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model
|
|||
store,
|
||||
subscription.ID,
|
||||
subscription.SiteURL,
|
||||
feedCreationRequest.UserAgent,
|
||||
feedCreationRequest.FetchViaProxy,
|
||||
feedCreationRequest.AllowSelfSignedCertificates,
|
||||
)
|
||||
|
@ -178,6 +179,7 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64) error {
|
|||
store,
|
||||
originalFeed.ID,
|
||||
originalFeed.SiteURL,
|
||||
originalFeed.UserAgent,
|
||||
originalFeed.FetchViaProxy,
|
||||
originalFeed.AllowSelfSignedCertificates,
|
||||
)
|
||||
|
@ -196,9 +198,9 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func checkFeedIcon(store *storage.Storage, feedID int64, websiteURL string, fetchViaProxy, allowSelfSignedCertificates bool) {
|
||||
func checkFeedIcon(store *storage.Storage, feedID int64, websiteURL, userAgent string, fetchViaProxy, allowSelfSignedCertificates bool) {
|
||||
if !store.HasIcon(feedID) {
|
||||
icon, err := icon.FindIcon(websiteURL, fetchViaProxy, allowSelfSignedCertificates)
|
||||
icon, err := icon.FindIcon(websiteURL, userAgent, fetchViaProxy, allowSelfSignedCertificates)
|
||||
if err != nil {
|
||||
logger.Debug(`[CheckFeedIcon] %v (feedID=%d websiteURL=%s)`, err, feedID, websiteURL)
|
||||
} else if icon == nil {
|
||||
|
|
|
@ -21,21 +21,25 @@ import (
|
|||
)
|
||||
|
||||
// FindIcon try to find the website's icon.
|
||||
func FindIcon(websiteURL string, fetchViaProxy, allowSelfSignedCertificates bool) (*model.Icon, error) {
|
||||
func FindIcon(websiteURL, userAgent string, fetchViaProxy, allowSelfSignedCertificates bool) (*model.Icon, error) {
|
||||
rootURL := url.RootURL(websiteURL)
|
||||
logger.Debug("[FindIcon] Trying to find an icon: rootURL=%q websiteURL=%q userAgent=%q", rootURL, websiteURL, userAgent)
|
||||
|
||||
clt := client.NewClientWithConfig(rootURL, config.Opts)
|
||||
clt.WithUserAgent(userAgent)
|
||||
clt.AllowSelfSignedCertificates = allowSelfSignedCertificates
|
||||
|
||||
if fetchViaProxy {
|
||||
clt.WithProxy()
|
||||
}
|
||||
|
||||
response, err := clt.Get()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to download website index page: %v", err)
|
||||
return nil, fmt.Errorf("icon: unable to download website index page: %v", err)
|
||||
}
|
||||
|
||||
if response.HasServerFailure() {
|
||||
return nil, fmt.Errorf("unable to download website index page: status=%d", response.StatusCode)
|
||||
return nil, fmt.Errorf("icon: unable to download website index page: status=%d", response.StatusCode)
|
||||
}
|
||||
|
||||
iconURL, err := parseDocument(rootURL, response.Body)
|
||||
|
@ -48,7 +52,7 @@ func FindIcon(websiteURL string, fetchViaProxy, allowSelfSignedCertificates bool
|
|||
}
|
||||
|
||||
logger.Debug("[FindIcon] Fetching icon => %s", iconURL)
|
||||
icon, err := downloadIcon(iconURL, fetchViaProxy, allowSelfSignedCertificates)
|
||||
icon, err := downloadIcon(iconURL, userAgent, fetchViaProxy, allowSelfSignedCertificates)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
@ -66,7 +70,7 @@ func parseDocument(websiteURL string, data io.Reader) (string, error) {
|
|||
|
||||
doc, err := goquery.NewDocumentFromReader(data)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("unable to read document: %v", err)
|
||||
return "", fmt.Errorf("icon: unable to read document: %v", err)
|
||||
}
|
||||
|
||||
var iconURL string
|
||||
|
@ -91,28 +95,30 @@ func parseDocument(websiteURL string, data io.Reader) (string, error) {
|
|||
return iconURL, nil
|
||||
}
|
||||
|
||||
func downloadIcon(iconURL string, fetchViaProxy, allowSelfSignedCertificates bool) (*model.Icon, error) {
|
||||
func downloadIcon(iconURL, userAgent string, fetchViaProxy, allowSelfSignedCertificates bool) (*model.Icon, error) {
|
||||
clt := client.NewClientWithConfig(iconURL, config.Opts)
|
||||
clt.WithUserAgent(userAgent)
|
||||
clt.AllowSelfSignedCertificates = allowSelfSignedCertificates
|
||||
if fetchViaProxy {
|
||||
clt.WithProxy()
|
||||
}
|
||||
|
||||
response, err := clt.Get()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to download iconURL: %v", err)
|
||||
return nil, fmt.Errorf("icon: unable to download iconURL: %v", err)
|
||||
}
|
||||
|
||||
if response.HasServerFailure() {
|
||||
return nil, fmt.Errorf("unable to download icon: status=%d", response.StatusCode)
|
||||
return nil, fmt.Errorf("icon: unable to download icon: status=%d", response.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(response.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("unable to read downloaded icon: %v", err)
|
||||
return nil, fmt.Errorf("icon: unable to read downloaded icon: %v", err)
|
||||
}
|
||||
|
||||
if len(body) == 0 {
|
||||
return nil, fmt.Errorf("downloaded icon is empty, iconURL=%s", iconURL)
|
||||
return nil, fmt.Errorf("icon: downloaded icon is empty, iconURL=%s", iconURL)
|
||||
}
|
||||
|
||||
icon := &model.Icon{
|
||||
|
|
|
@ -60,6 +60,7 @@ func (h *handler) imageProxy(w http.ResponseWriter, r *http.Request) {
|
|||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
logger.Error(`[Proxy] code=%d url=%q`, resp.StatusCode, imageURL)
|
||||
html.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue