forks/archivebox
1
0
Fork 0
Code Releases Activity

15042020

Browse source
This commit is contained in:
misha 2020-04-15 11:54:53 +03:00
parent bb580533f7
commit 1aa2a5b069
3 changed files with 4 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
archivebox/archive_methods.py
View file

@ -30,6 +30,7 @@ from config import (
OUTPUT_DIR, OUTPUT_DIR,
GIT_DOMAINS, GIT_DOMAINS,
GIT_SHA, GIT_SHA,
RESTRICT_FILE_NAMES,
CURL_USER_AGENT, CURL_USER_AGENT,
WGET_USER_AGENT, WGET_USER_AGENT,
CHECK_SSL_VALIDITY, CHECK_SSL_VALIDITY,
@ -227,7 +228,7 @@ def fetch_wget(link_dir, link, timeout=TIMEOUT):
'--span-hosts', '--span-hosts',
'--no-parent', '--no-parent',
'-e', 'robots=off', '-e', 'robots=off',
'--restrict-file-names=nocontrol', *(('--restrict-file-names={}'.format(RESTRICT_FILE_NAMES),) if RESTRICT_FILE_NAMES else ()),
'--timeout={}'.format(timeout), '--timeout={}'.format(timeout),
*(('--compression=auto',) if WGET_AUTO_COMPRESSION else ()), *(('--compression=auto',) if WGET_AUTO_COMPRESSION else ()),
*(() if FETCH_WARC else ('--timestamping',)), *(() if FETCH_WARC else ('--timestamping',)),

1
archivebox/config.py
View file

@ -34,6 +34,7 @@ SUBMIT_ARCHIVE_DOT_ORG = os.getenv('SUBMIT_ARCHIVE_DOT_ORG', 'True'
CHECK_SSL_VALIDITY = os.getenv('CHECK_SSL_VALIDITY', 'True' ).lower() == 'true' CHECK_SSL_VALIDITY = os.getenv('CHECK_SSL_VALIDITY', 'True' ).lower() == 'true'
RESOLUTION = os.getenv('RESOLUTION', '1440,2000' ) RESOLUTION = os.getenv('RESOLUTION', '1440,2000' )
RESTRICT_FILE_NAMES = os.getenv('RESTRICT_FILE_NAMES', 'windows' )
GIT_DOMAINS = os.getenv('GIT_DOMAINS', 'github.com,bitbucket.org,gitlab.com').split(',') GIT_DOMAINS = os.getenv('GIT_DOMAINS', 'github.com,bitbucket.org,gitlab.com').split(',')
CURL_USER_AGENT = os.getenv('CURL_USER_AGENT', 'ArchiveBox/{GIT_SHA} (+https://github.com/pirate/ArchiveBox/)') CURL_USER_AGENT = os.getenv('CURL_USER_AGENT', 'ArchiveBox/{GIT_SHA} (+https://github.com/pirate/ArchiveBox/)')
WGET_USER_AGENT = os.getenv('WGET_USER_AGENT', 'ArchiveBox/{GIT_SHA} (+https://github.com/pirate/ArchiveBox/) wget/{WGET_VERSION}') WGET_USER_AGENT = os.getenv('WGET_USER_AGENT', 'ArchiveBox/{GIT_SHA} (+https://github.com/pirate/ArchiveBox/) wget/{WGET_VERSION}')

1
etc/ArchiveBox.conf.default
View file

@ -39,6 +39,7 @@
#CHECK_SSL_VALIDITY=True #CHECK_SSL_VALIDITY=True
#FETCH_WGET_REQUISITES=True #FETCH_WGET_REQUISITES=True
#RESTRICT_FILE_NAMES="windows"
#RESOLUTION="1440,900" #RESOLUTION="1440,900"
#CURL_USER_AGENT="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36" #CURL_USER_AGENT="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36"
#WGET_USER_AGENT="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36" #WGET_USER_AGENT="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.75 Safari/537.36"