forks/archivebox
1
0
Fork 0
Code Releases Activity

After a timeout, chrome will leave behind a SingletonLock, which prevents future instances of chrome from starting. When an extractor fails due to a timeout, remove this file.

Browse source
This commit is contained in:
spresse1 2023-08-28 17:27:03 +02:00
parent 00ecf57b0f
commit 603ce7ec10
4 changed files with 18 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
archivebox/extractors/dom.py
View file

@ -9,6 +9,7 @@ from ..util import (
enforce_types, enforce_types,
is_static_file, is_static_file,
chrome_args, chrome_args,
chrome_cleanup,
) )
from ..config import ( from ..config import (
TIMEOUT, TIMEOUT,
@ -57,6 +58,7 @@ def save_dom(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
except Exception as err: except Exception as err:
status = 'failed' status = 'failed'
output = err output = err
chrome_cleanup()
finally: finally:
timer.end() timer.end()

2
archivebox/extractors/pdf.py
View file

@ -9,6 +9,7 @@ from ..util import (
enforce_types, enforce_types,
is_static_file, is_static_file,
chrome_args, chrome_args,
chrome_cleanup,
) )
from ..config import ( from ..config import (
TIMEOUT, TIMEOUT,
@ -54,6 +55,7 @@ def save_pdf(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) ->
except Exception as err: except Exception as err:
status = 'failed' status = 'failed'
output = err output = err
chrome_cleanup()
finally: finally:
timer.end() timer.end()

2
archivebox/extractors/screenshot.py
View file

@ -9,6 +9,7 @@ from ..util import (
enforce_types, enforce_types,
is_static_file, is_static_file,
chrome_args, chrome_args,
chrome_cleanup,
) )
from ..config import ( from ..config import (
TIMEOUT, TIMEOUT,
@ -54,6 +55,7 @@ def save_screenshot(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEO
except Exception as err: except Exception as err:
status = 'failed' status = 'failed'
output = err output = err
chrome_cleanup()
finally: finally:
timer.end() timer.end()

12
archivebox/util.py
View file

@ -17,6 +17,8 @@ from requests.exceptions import RequestException, ReadTimeout
from .vendor.base32_crockford import encode as base32_encode # type: ignore from .vendor.base32_crockford import encode as base32_encode # type: ignore
from w3lib.encoding import html_body_declared_encoding, http_content_type_encoding from w3lib.encoding import html_body_declared_encoding, http_content_type_encoding
from os.path import lexists
from os import remove as remove_file
try: try:
import chardet import chardet
@ -272,6 +274,16 @@ def chrome_args(**options) -> List[str]:
return cmd_args return cmd_args
def chrome_cleanup():
"""
Cleans up any state or runtime files that chrome leaves behind when killed by
a timeout or other error
"""
from .config import IN_DOCKER
if IN_DOCKER and lexists("/home/archivebox/.config/chromium/SingletonLock"):
remove_file("/home/archivebox/.config/chromium/SingletonLock")
def ansi_to_html(text): def ansi_to_html(text):
""" """