test: add basic download_url test

tests/test_util.py

@@ -1,21 +1,5 @@
-#@enforce_types
+from archivebox import util
-#def download_url(url: str, timeout: int=None) -> str:
+
-#    """Download the contents of a remote url and return the text"""
+def test_download_url_downloads_content():
-#    from .config import TIMEOUT, CHECK_SSL_VALIDITY, WGET_USER_AGENT
+    text = util.download_url("https://example.com")
-#    timeout = timeout or TIMEOUT
+    assert "Example Domain" in text
-#    response = requests.get(
-#        url,
-#        headers={'User-Agent': WGET_USER_AGENT},
-#        verify=CHECK_SSL_VALIDITY,
-#        timeout=timeout,
-#    )
-#    if response.headers.get('Content-Type') == 'application/rss+xml':
-#        # Based on https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py
-#        _TEMPLATE = r'''%s\s*=\s*["']?\s*%s\s*["']?'''
-#        _XML_ENCODING_RE = _TEMPLATE % ('encoding', r'(?P<xmlcharset>[\w-]+)')
-#        _BODY_ENCODING_PATTERN = r'<\s*(\?xml\s[^>]+%s)' % (_XML_ENCODING_RE)
-#        _BODY_ENCODING_STR_RE = re.compile(_BODY_ENCODING_PATTERN, re.I | re.VERBOSE)
-#        match = _BODY_ENCODING_STR_RE.search(response.text[:1024])
-#        if match:
-#            response.encoding = match.group('xmlcharset')
-#    return response.text