Inline a one-liner function
No need to expose a symbol for this.
This commit is contained in:
parent
6bc819e198
commit
fc4bdf3ab0
4 changed files with 7 additions and 12 deletions
|
@ -35,8 +35,3 @@ func CharsetReader(charsetLabel string, input io.Reader) (io.Reader, error) {
|
||||||
// Transform document to UTF-8 from the specified encoding in XML prolog.
|
// Transform document to UTF-8 from the specified encoding in XML prolog.
|
||||||
return charset.NewReaderLabel(charsetLabel, r)
|
return charset.NewReaderLabel(charsetLabel, r)
|
||||||
}
|
}
|
||||||
|
|
||||||
// CharsetReaderFromContentType is used when the encoding is not specified for the input document.
|
|
||||||
func CharsetReaderFromContentType(contentType string, input io.Reader) (io.Reader, error) {
|
|
||||||
return charset.NewReader(input, contentType)
|
|
||||||
}
|
|
||||||
|
|
|
@ -15,11 +15,11 @@ import (
|
||||||
"miniflux.app/v2/internal/config"
|
"miniflux.app/v2/internal/config"
|
||||||
"miniflux.app/v2/internal/crypto"
|
"miniflux.app/v2/internal/crypto"
|
||||||
"miniflux.app/v2/internal/model"
|
"miniflux.app/v2/internal/model"
|
||||||
"miniflux.app/v2/internal/reader/encoding"
|
|
||||||
"miniflux.app/v2/internal/reader/fetcher"
|
"miniflux.app/v2/internal/reader/fetcher"
|
||||||
"miniflux.app/v2/internal/urllib"
|
"miniflux.app/v2/internal/urllib"
|
||||||
|
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
|
"golang.org/x/net/html/charset"
|
||||||
)
|
)
|
||||||
|
|
||||||
type IconFinder struct {
|
type IconFinder struct {
|
||||||
|
@ -191,7 +191,7 @@ func findIconURLsFromHTMLDocument(body io.Reader, contentType string) ([]string,
|
||||||
"link[rel='apple-touch-icon-precomposed.png']",
|
"link[rel='apple-touch-icon-precomposed.png']",
|
||||||
}
|
}
|
||||||
|
|
||||||
htmlDocumentReader, err := encoding.CharsetReaderFromContentType(contentType, body)
|
htmlDocumentReader, err := charset.NewReader(body, contentType)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("icon: unable to create charset reader: %w", err)
|
return nil, fmt.Errorf("icon: unable to create charset reader: %w", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,12 +10,12 @@ import (
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"miniflux.app/v2/internal/config"
|
"miniflux.app/v2/internal/config"
|
||||||
"miniflux.app/v2/internal/reader/encoding"
|
|
||||||
"miniflux.app/v2/internal/reader/fetcher"
|
"miniflux.app/v2/internal/reader/fetcher"
|
||||||
"miniflux.app/v2/internal/reader/readability"
|
"miniflux.app/v2/internal/reader/readability"
|
||||||
"miniflux.app/v2/internal/urllib"
|
"miniflux.app/v2/internal/urllib"
|
||||||
|
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
|
"golang.org/x/net/html/charset"
|
||||||
)
|
)
|
||||||
|
|
||||||
func ScrapeWebsite(requestBuilder *fetcher.RequestBuilder, websiteURL, rules string) (string, error) {
|
func ScrapeWebsite(requestBuilder *fetcher.RequestBuilder, websiteURL, rules string) (string, error) {
|
||||||
|
@ -42,9 +42,9 @@ func ScrapeWebsite(requestBuilder *fetcher.RequestBuilder, websiteURL, rules str
|
||||||
var content string
|
var content string
|
||||||
var err error
|
var err error
|
||||||
|
|
||||||
htmlDocumentReader, err := encoding.CharsetReaderFromContentType(
|
htmlDocumentReader, err := charset.NewReader(
|
||||||
responseHandler.ContentType(),
|
|
||||||
responseHandler.Body(config.Opts.HTTPClientMaxBodySize()),
|
responseHandler.Body(config.Opts.HTTPClientMaxBodySize()),
|
||||||
|
responseHandler.ContentType(),
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("scraper: unable to read HTML document: %v", err)
|
return "", fmt.Errorf("scraper: unable to read HTML document: %v", err)
|
||||||
|
|
|
@ -14,12 +14,12 @@ import (
|
||||||
"miniflux.app/v2/internal/integration/rssbridge"
|
"miniflux.app/v2/internal/integration/rssbridge"
|
||||||
"miniflux.app/v2/internal/locale"
|
"miniflux.app/v2/internal/locale"
|
||||||
"miniflux.app/v2/internal/model"
|
"miniflux.app/v2/internal/model"
|
||||||
"miniflux.app/v2/internal/reader/encoding"
|
|
||||||
"miniflux.app/v2/internal/reader/fetcher"
|
"miniflux.app/v2/internal/reader/fetcher"
|
||||||
"miniflux.app/v2/internal/reader/parser"
|
"miniflux.app/v2/internal/reader/parser"
|
||||||
"miniflux.app/v2/internal/urllib"
|
"miniflux.app/v2/internal/urllib"
|
||||||
|
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
|
"golang.org/x/net/html/charset"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
@ -150,7 +150,7 @@ func (f *SubscriptionFinder) FindSubscriptionsFromWebPage(websiteURL, contentTyp
|
||||||
"link[type='application/feed+json']": parser.FormatJSON,
|
"link[type='application/feed+json']": parser.FormatJSON,
|
||||||
}
|
}
|
||||||
|
|
||||||
htmlDocumentReader, err := encoding.CharsetReaderFromContentType(contentType, body)
|
htmlDocumentReader, err := charset.NewReader(body, contentType)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, locale.NewLocalizedErrorWrapper(err, "error.unable_to_parse_html_document", err)
|
return nil, locale.NewLocalizedErrorWrapper(err, "error.unable_to_parse_html_document", err)
|
||||||
}
|
}
|
||||||
|
|
Loading…
Add table
Reference in a new issue