add DOM archiving with chrome headless

archive_methods.py

@@ -17,6 +17,7 @@ from config import (
     FETCH_WGET_REQUISITES,
     FETCH_PDF,
     FETCH_SCREENSHOT,
+    FETCH_DOM,
     RESOLUTION,
     CHECK_SSL_VALIDITY,
     SUBMIT_ARCHIVE_DOT_ORG,
@@ -93,6 +94,9 @@ def archive_link(link_dir, link, overwrite=True):
     if FETCH_SCREENSHOT:
         link = fetch_screenshot(link_dir, link, overwrite=overwrite)
 
+    if FETCH_DOM:
+        link = fetch_dom(link_dir, link, overwrite=overwrite)
+
     if SUBMIT_ARCHIVE_DOT_ORG:
         link = archive_dot_org(link_dir, link, overwrite=overwrite)
 
@@ -252,7 +256,6 @@ def fetch_pdf(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DI
         'output': output,
     }
 
-
 @attach_result_to_link('screenshot')
 def fetch_screenshot(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DIR, resolution=RESOLUTION):
     """take screenshot of site using chrome --headless"""
@@ -289,6 +292,43 @@ def fetch_screenshot(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_
         'output': output,
     }
     
+@attach_result_to_link('dom')
+def fetch_dom(link_dir, link, timeout=TIMEOUT, user_data_dir=CHROME_USER_DATA_DIR):
+    """print HTML of site to file using chrome --dump-html"""
+
+    if link['type'] in ('PDF', 'image'):
+        return {'output': wget_output_path(link)}
+    
+    output_path = os.path.join(link_dir, 'output.html')
+
+    if os.path.exists(output_path):
+        return {'output': 'output.html', 'status': 'skipped'}
+
+    CMD = [
+        *chrome_headless(user_data_dir=user_data_dir),
+        '--dump-dom',
+        link['url']
+    ]
+    end = progress(timeout, prefix='      ')
+    try:
+        with open(output_path, 'w+') as f:
+            result = run(CMD, stdout=f, stderr=PIPE, cwd=link_dir, timeout=timeout + 1)  # output.html
+        end()
+        if result.returncode:
+            print('     ', (result.stderr).decode())
+            raise Exception('Failed to fetch DOM')
+        chmod_file('output.html', cwd=link_dir)
+        output = 'output.html'
+    except Exception as e:
+        end()
+        print('        Run to see full output:', 'cd {}; {}'.format(link_dir, ' '.join(CMD)))
+        print('        {}Failed: {} {}{}'.format(ANSI['red'], e.__class__.__name__, e, ANSI['reset']))
+        output = e
+
+    return {
+        'cmd': CMD,
+        'output': output,
+    }
 
 @attach_result_to_link('archive_org')
 def archive_dot_org(link_dir, link, timeout=TIMEOUT):
@@ -445,7 +485,7 @@ def fetch_favicon(link_dir, link, timeout=TIMEOUT):
 
 
 def chrome_headless(binary=CHROME_BINARY, user_data_dir=CHROME_USER_DATA_DIR):
-    args = [binary, '--headless', '--disable-gpu']
+    args = [binary, '--headless']  # '--disable-gpu'
     default_profile = os.path.expanduser('~/Library/Application Support/Google/Chrome/Default')
     if user_data_dir:
         args.append('--user-data-dir={}'.format(user_data_dir))

config.py

@@ -19,6 +19,7 @@ FETCH_AUDIO =            os.getenv('FETCH_AUDIO',            'False'
 FETCH_VIDEO =            os.getenv('FETCH_VIDEO',            'False'            ).lower() == 'true'
 FETCH_PDF =              os.getenv('FETCH_PDF',              'True'             ).lower() == 'true'
 FETCH_SCREENSHOT =       os.getenv('FETCH_SCREENSHOT',       'True'             ).lower() == 'true'
+FETCH_DOM =              os.getenv('FETCH_DOM',              'True'             ).lower() == 'true'
 FETCH_FAVICON =          os.getenv('FETCH_FAVICON',          'True'             ).lower() == 'true'
 SUBMIT_ARCHIVE_DOT_ORG = os.getenv('SUBMIT_ARCHIVE_DOT_ORG', 'True'             ).lower() == 'true'
 RESOLUTION =             os.getenv('RESOLUTION',             '1440,1200'        )

links.py

@@ -68,6 +68,9 @@ def validate_links(links):
         if not link['latest'].get('screenshot'):
             link['latest']['screenshot'] = None
 
+        if not link['latest'].get('dom'):
+            link['latest']['dom'] = None
+
     return list(links)
 
 

templates/index.html

@@ -113,6 +113,7 @@
                     <th style="width: 2vw;">Status</th>
                     <th style="width: 43vw;">Saved Articles ($num_links)</th>
                     <th style="width: 50px">Index</th>
+                    <th style="width: 50px">HTML</th>
                     <th style="width: 50px">PDF</th>
                     <th style="width: 60px;font-size:0.8em;">Screenshot</th>
                     <th style="width: 50px">A.org</th>

templates/index_row.html

@@ -5,12 +5,13 @@
             <img src="$favicon_url" onerror="this.src='static/spinner.gif'" class="link-favicon">
         </a>
     </td>
-    <td><a href="$archive_url" style="font-size:1.4em;text-decoration:none;color:black;" title="$title">
+    <td style="text-align: left"><a href="$archive_url" style="font-size:1.4em;text-decoration:none;color:black;" title="$title">
         $title <small style="background-color: #eee;border-radius:4px; float:right">$tags</small>
     </td>
-    <td style="text-align:center"><a href="$files_url" title="Files">📂</a></td>
+    <td><a href="$files_url" title="Files">📂</a></td>
-    <td style="text-align:center"><a href="$pdf_link" title="PDF">📄</a></td>
+    <td><a href="$dom_link" title="HTML">📄</a></td>
-    <td style="text-align:center"><a href="$screenshot_link" title="Screenshot">🖼</a></td>
+    <td><a href="$pdf_link" title="PDF">📜</a></td>
-    <td style="text-align:center"><a href="$archive_org_url" title="Archive.org">🏛</a></td>
+    <td><a href="$screenshot_link" title="Screenshot">🖼</a></td>
-    <td>🔗 <img src="$google_favicon_url" height="16px"> <a href="$url">$url</a></td>
+    <td><a href="$archive_org_url" title="Archive.org">🏛</a></td>
+    <td style="text-align: left"><!--🔗 <img src="$google_favicon_url" height="16px">--> <a href="$url">$url</a></td>
 </tr>

util.py

@@ -23,6 +23,7 @@ from config import (
     FETCH_WGET,
     FETCH_PDF,
     FETCH_SCREENSHOT,
+    FETCH_DOM,
     FETCH_FAVICON,
     FETCH_AUDIO,
     FETCH_VIDEO,
@@ -49,7 +50,7 @@ def check_dependencies():
         print('    See https://github.com/pirate/bookmark-archiver#troubleshooting for help upgrading your Python installation.')
         raise SystemExit(1)
 
-    if FETCH_PDF or FETCH_SCREENSHOT:
+    if FETCH_PDF or FETCH_SCREENSHOT or FETCH_DOM:
         if run(['which', CHROME_BINARY], stdout=DEVNULL).returncode:
             print('{}[X] Missing dependency: {}{}'.format(ANSI['red'], CHROME_BINARY, ANSI['reset']))
             print('    Run ./setup.sh, then confirm it was installed with: {} --version'.format(CHROME_BINARY))
@@ -64,7 +65,7 @@ def check_dependencies():
             version = [l for l in version_lines if l.isdigit()][-1]
             if int(version) < 59:
                 print(version_lines)
-                print('{red}[X] Chrome version must be 59 or greater for headless PDF and screenshot saving{reset}'.format(**ANSI))
+                print('{red}[X] Chrome version must be 59 or greater for headless PDF, screenshot, and DOM saving{reset}'.format(**ANSI))
                 print('    See https://github.com/pirate/bookmark-archiver for help.')
                 raise SystemExit(1)
         except (IndexError, TypeError, OSError):
@@ -459,6 +460,7 @@ def derived_link_info(link):
         'archive_url': 'archive/{}/{}'.format(link['timestamp'], wget_output_path(link)),
         'pdf_link': 'archive/{timestamp}/output.pdf'.format(**link),
         'screenshot_link': 'archive/{timestamp}/screenshot.png'.format(**link),
+        'dom_link': 'archive/{timestamp}/output.html'.format(**link),
         'archive_org_url': 'https://web.archive.org/web/{base_url}'.format(**link),
     }
 
@@ -469,6 +471,7 @@ def derived_link_info(link):
             'archive_url': 'archive/{timestamp}/{base_url}'.format(**link),
             'pdf_link': 'archive/{timestamp}/{base_url}'.format(**link),
             'screenshot_link': 'archive/{timestamp}/{base_url}'.format(**link),
+            'dom_link': 'archive/{timestamp}/{base_url}'.format(**link),
             'title': '{title} ({type})'.format(**link),
         })
     return link_info