forks/archivebox
1
0
Fork 0
Code Releases Activity

put git clones in a git folder to avoid retries

Browse source
This commit is contained in:
Nick Sweeting 2019-02-27 15:55:39 -05:00
parent 3eaa76267e
commit 552734241b
1 changed files with 20 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
archivebox/archive_methods.py
View file

@ -43,6 +43,7 @@ from config import (
) )
from util import ( from util import (
domain, domain,
without_query,
without_fragment, without_fragment,
fetch_page_title, fetch_page_title,
progress, progress,
@ -589,30 +590,38 @@ def fetch_media(link_dir, link, timeout=MEDIA_TIMEOUT, overwrite=False):
def fetch_git(link_dir, link, timeout=TIMEOUT): def fetch_git(link_dir, link, timeout=TIMEOUT):
"""download full site using git""" """download full site using git"""
if not (domain(link['url']) in GIT_DOMAINS url_is_clonable = (
or link['url'].endswith('.git') domain(link['url']) in GIT_DOMAINS
or link['type'] == 'git'): or link['url'].endswith('.git')
return or link['type'] == 'git'
)
if os.path.exists(os.path.join(link_dir, 'git')): if not url_is_clonable:
return {'output': None, 'status': 'skipped'}
git_dir = os.path.join(link_dir, 'git')
if os.path.exists(git_dir):
return {'output': 'git', 'status': 'skipped'} return {'output': 'git', 'status': 'skipped'}
os.makedirs(git_dir, exist_ok=True)
output = 'git'
CMD = [ CMD = [
GIT_BINARY, GIT_BINARY,
*(() if CHECK_SSL_VALIDITY else ('-c', 'http.sslVerify=false')),
'clone', 'clone',
'--mirror', '--mirror',
'--recursive', '--recursive',
without_fragment(link['url']), *(() if CHECK_SSL_VALIDITY else ('-c', 'http.sslVerify=false')),
without_query(without_fragment(link['url'])),
] ]
output = 'git'
end = progress(timeout, prefix=' ') end = progress(timeout, prefix=' ')
try: try:
result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=link_dir, timeout=timeout + 1) # git/<reponame> result = run(CMD, stdout=PIPE, stderr=PIPE, cwd=git_dir, timeout=timeout + 1) # git/<reponame>
end() end()
if result.returncode > 0: if result.returncode == 128:
# ignore failed re-download when the folder already exists
pass
elif result.returncode > 0:
print(' got git response code {}:'.format(result.returncode)) print(' got git response code {}:'.format(result.returncode))
raise Exception('Failed git download') raise Exception('Failed git download')
except Exception as e: except Exception as e: