forks/archivebox
1
0
Fork 0
Code Releases Activity

Merge pull request #1146 from mrled/add-favicon-provider

Browse source
This commit is contained in:
Nick Sweeting 2023-05-05 19:00:15 -07:00 committed by GitHub
commit 5bbc5228cc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 6 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
archivebox/config.py
View file

@ -183,7 +183,8 @@ CONFIG_SCHEMA: Dict[str, ConfigDefaultDict] = {
'--compressed'
]},
'GIT_ARGS': {'type': list, 'default': ['--recursive']},
'SINGLEFILE_ARGS': {'type': list, 'default' : None}
'SINGLEFILE_ARGS': {'type': list, 'default' : None},
'FAVICON_PROVIDER': {'type': str, 'default': 'https://www.google.com/s2/favicons?domain={}'},
},
'SEARCH_BACKEND_CONFIG' : {

3
archivebox/extractors/favicon.py
View file

@ -10,6 +10,7 @@ from ..util import enforce_types, domain
from ..config import (
TIMEOUT,
SAVE_FAVICON,
FAVICON_PROVIDER,
CURL_BINARY,
CURL_ARGS,
CURL_VERSION,
@ -40,7 +41,7 @@ def save_favicon(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT)
'--output', str(output),
*(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
*([] if CHECK_SSL_VALIDITY else ['--insecure']),
'https://www.google.com/s2/favicons?domain={}'.format(domain(link.url)),
FAVICON_PROVIDER.format(domain(link.url)),
]
status = 'failed'
timer = TimedProgress(timeout, prefix=' ')

4
archivebox/index/schema.py
View file

@ -20,7 +20,7 @@ from django.utils.functional import cached_property
from ..system import get_dir_size
from ..util import ts_to_date_str, parse_date
from ..config import OUTPUT_DIR, ARCHIVE_DIR_NAME
from ..config import OUTPUT_DIR, ARCHIVE_DIR_NAME, FAVICON_PROVIDER
class ArchiveError(Exception):
def __init__(self, message, hints=None):
@ -423,7 +423,7 @@ class Link:
canonical = {
'index_path': 'index.html',
'favicon_path': 'favicon.ico',
'google_favicon_path': 'https://www.google.com/s2/favicons?domain={}'.format(self.domain),
'google_favicon_path': FAVICON_PROVIDER.format(self.domain),
'wget_path': wget_output_path(self),
'warc_path': 'warc/',
'singlefile_path': 'singlefile.html',