forks/archivebox
1
0
Fork 0
Code Releases Activity

add DOM archiving with chrome headless

Browse source
This commit is contained in:
Nick Sweeting 2018-06-10 18:45:41 -04:00
parent 2e19050aba
commit f78838ef40
6 changed files with 59 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
archive_methods.py
View file

@ -17,6 +17,7 @@ from config import (
FETCH_WGET_REQUISITES,
FETCH_PDF,
FETCH_SCREENSHOT,
FETCH_DOM,
RESOLUTION,
CHECK_SSL_VALIDITY,
SUBMIT_ARCHIVE_DOT_ORG,
@ -93,6 +94,9 @@ def archive_link(link_dir, link, overwrite=True):
if FETCH_SCREENSHOT:
link = fetch_screenshot(link_dir, link, overwrite=overwrite)
if FETCH_DOM:
link = fetch_dom(link_dir, link, overwrite=overwrite)
if SUBMIT_ARCHIVE_DOT_ORG:
link = archive_dot_org(link_dir, link, overwrite=overwrite)
@ -252,7 +256,6 @@ def fetch_pdf(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DI
'output': output,
}
@attach_result_to_link('screenshot')
def fetch_screenshot(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DIR, resolution=RESOLUTION):
"""take screenshot of site using chrome --headless"""
@ -289,6 +292,43 @@ def fetch_screenshot(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_
'output': output,
}
@attach_result_to_link('dom')
def fetch_dom(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DIR):
"""print HTML of site to file using chrome --dump-html"""
if link['type'] in ('PDF', 'image'):
return {'output': wget_output_path(link)}
output_path = os.path.join(link_dir, 'output.html')
if os.path.exists(output_path):
return {'output': 'output.html', 'status': 'skipped'}
CMD = [
*chrome_headless(user_data_dir=user_data_dir),
'--dump-dom',
link['url']
]
end = progress(timeout, prefix=' ')
try:
with open(output_path, 'w+') as f:
result = run(CMD, stdout=f, stderr=PIPE, cwd=link_dir, timeout=timeout + 1) # output.html
end()
if result.returncode:
print(' ', (result.stderr).decode())
raise Exception('Failed to fetch DOM')
chmod_file('output.html', cwd=link_dir)
output = 'output.html'
except Exception as e:
end()
print(' Run to see full output:', 'cd {}; {}'.format(link_dir, ' '.join(CMD)))
print(' {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
output = e
return {
'cmd': CMD,
'output': output,
}
@attach_result_to_link('archive_org')
def archive_dot_org(link_dir, link, timeout=TIMEOUT):
@ -445,7 +485,7 @@ def fetch_favicon(link_dir, link, timeout=TIMEOUT):
def chrome_headless(binary=CHROME_BINARY, user_data_dir=CHROME_USER_DATA_DIR):
args = [binary, '--headless', '--disable-gpu']
args = [binary, '--headless'] # '--disable-gpu'
default_profile = os.path.expanduser('~/Library/Application Support/Google/Chrome/Default')
if user_data_dir:
args.append('--user-data-dir={}'.format(user_data_dir))

1
config.py
View file

@ -19,6 +19,7 @@ FETCH_AUDIO = os.getenv('FETCH_AUDIO', 'False'
FETCH_VIDEO = os.getenv('FETCH_VIDEO', 'False' ).lower() == 'true'
FETCH_PDF = os.getenv('FETCH_PDF', 'True' ).lower() == 'true'
FETCH_SCREENSHOT = os.getenv('FETCH_SCREENSHOT', 'True' ).lower() == 'true'
FETCH_DOM = os.getenv('FETCH_DOM', 'True' ).lower() == 'true'
FETCH_FAVICON = os.getenv('FETCH_FAVICON', 'True' ).lower() == 'true'
SUBMIT_ARCHIVE_DOT_ORG = os.getenv('SUBMIT_ARCHIVE_DOT_ORG', 'True' ).lower() == 'true'
RESOLUTION = os.getenv('RESOLUTION', '1440,1200' )

3
links.py
View file

@ -68,6 +68,9 @@ def validate_links(links):
if not link['latest'].get('screenshot'):
link['latest']['screenshot'] = None
if not link['latest'].get('dom'):
link['latest']['dom'] = None
return list(links)

1
templates/index.html
View file

@ -113,6 +113,7 @@
<th style="width: 2vw;">Status</th>
<th style="width: 43vw;">Saved Articles ($num_links)</th>
<th style="width: 50px">Index</th>
<th style="width: 50px">HTML</th>
<th style="width: 50px">PDF</th>
<th style="width: 60px;font-size:0.8em;">Screenshot</th>
<th style="width: 50px">A.org</th>

13
templates/index_row.html
View file

@ -5,12 +5,13 @@
<img src="$favicon_url" onerror="this.src='static/spinner.gif'" class="link-favicon">
</a>
</td>
<td><a href="$archive_url" style="font-size:1.4em;text-decoration:none;color:black;" title="$title">
<td style="text-align: left"><a href="$archive_url" style="font-size:1.4em;text-decoration:none;color:black;" title="$title">
$title <small style="background-color: #eee;border-radius:4px; float:right">$tags</small>
</td>
<td style="text-align:center"><a href="$files_url" title="Files">📂</a></td>
<td style="text-align:center"><a href="$pdf_link" title="PDF">📄</a></td>
<td style="text-align:center"><a href="$screenshot_link" title="Screenshot">🖼</a></td>
<td style="text-align:center"><a href="$archive_org_url" title="Archive.org">🏛</a></td>
<td>🔗 <img src="$google_favicon_url" height="16px"> <a href="$url">$url</a></td>
<td><a href="$files_url" title="Files">📂</a></td>
<td><a href="$dom_link" title="HTML">📄</a></td>
<td><a href="$pdf_link" title="PDF">📜</a></td>
<td><a href="$screenshot_link" title="Screenshot">🖼</a></td>
<td><a href="$archive_org_url" title="Archive.org">🏛</a></td>
<td style="text-align: left"><!--🔗 <img src="$google_favicon_url" height="16px">--> <a href="$url">$url</a></td>
</tr>

7
util.py
View file

@ -23,6 +23,7 @@ from config import (
FETCH_WGET,
FETCH_PDF,
FETCH_SCREENSHOT,
FETCH_DOM,
FETCH_FAVICON,
FETCH_AUDIO,
FETCH_VIDEO,
@ -49,7 +50,7 @@ def check_dependencies():
print(' See https://github.com/pirate/bookmark-archiver#troubleshooting for help upgrading your Python installation.')
raise SystemExit(1)
if FETCH_PDF or FETCH_SCREENSHOT:
if FETCH_PDF or FETCH_SCREENSHOT or FETCH_DOM:
if run(['which', CHROME_BINARY], stdout=DEVNULL).returncode:
print('{}[X] Missing dependency: {}{}'.format(ANSI['red'], CHROME_BINARY, ANSI['reset']))
print(' Run ./setup.sh, then confirm it was installed with: {} --version'.format(CHROME_BINARY))
@ -64,7 +65,7 @@ def check_dependencies():
version = [l for l in version_lines if l.isdigit()][-1]
if int(version) < 59:
print(version_lines)
print('{red}[X] Chrome version must be 59 or greater for headless PDF and screenshot saving{reset}'.format(**ANSI))
print('{red}[X] Chrome version must be 59 or greater for headless PDF, screenshot, and DOM saving{reset}'.format(**ANSI))
print(' See https://github.com/pirate/bookmark-archiver for help.')
raise SystemExit(1)
except (IndexError, TypeError, OSError):
@ -459,6 +460,7 @@ def derived_link_info(link):
'archive_url': 'archive/{}/{}'.format(link['timestamp'], wget_output_path(link)),
'pdf_link': 'archive/{timestamp}/output.pdf'.format(**link),
'screenshot_link': 'archive/{timestamp}/screenshot.png'.format(**link),
'dom_link': 'archive/{timestamp}/output.html'.format(**link),
'archive_org_url': 'https://web.archive.org/web/{base_url}'.format(**link),
}
@ -469,6 +471,7 @@ def derived_link_info(link):
'archive_url': 'archive/{timestamp}/{base_url}'.format(**link),
'pdf_link': 'archive/{timestamp}/{base_url}'.format(**link),
'screenshot_link': 'archive/{timestamp}/{base_url}'.format(**link),
'dom_link': 'archive/{timestamp}/{base_url}'.format(**link),
'title': '{title} ({type})'.format(**link),
})
return link_info