From 63fe19e2c2d236cabae36ef441aff9fd46dd6014 Mon Sep 17 00:00:00 2001 From: Cristian Date: Fri, 3 Jul 2020 11:52:57 -0500 Subject: [PATCH 1/4] feat: Add pytest and initial tests --- setup.py | 3 +++ tests/test_init.py | 40 ++++++++++++++++++++++++++++++++++++++++ tests/test_util.py | 21 +++++++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100644 tests/test_init.py create mode 100644 tests/test_util.py diff --git a/setup.py b/setup.py index 049528fb..12002580 100755 --- a/setup.py +++ b/setup.py @@ -65,6 +65,9 @@ setuptools.setup( "sphinx-rtd-theme", "recommonmark", ], + "test": [ + "pytest" + ] # 'redis': ['redis', 'django-redis'], # 'pywb': ['pywb', 'redis'], }, diff --git a/tests/test_init.py b/tests/test_init.py new file mode 100644 index 00000000..b870a599 --- /dev/null +++ b/tests/test_init.py @@ -0,0 +1,40 @@ +# archivebox init +# archivebox add + +import os +import subprocess +from pathlib import Path +import json + +import pytest + +@pytest.fixture +def process(tmp_path): + os.chdir(tmp_path) + process = subprocess.run(['archivebox', 'init'], capture_output=True) + return process + + +def test_init(tmp_path, process): + assert "Initializing a new ArchiveBox collection in this folder..." in process.stdout.decode("utf-8") + +def test_update(tmp_path, process): + os.chdir(tmp_path) + update_process = subprocess.run(['archivebox', 'init'], capture_output=True) + assert "Updating existing ArchiveBox collection in this folder" in update_process.stdout.decode("utf-8") + +def test_add_link(tmp_path, process): + os.chdir(tmp_path) + add_process = subprocess.run(['archivebox', 'add', 'http://example.com'], capture_output=True) + archived_item_path = list(tmp_path.glob('archive/**/*'))[0] + + assert "index.json" in [x.name for x in archived_item_path.iterdir()] + + with open(archived_item_path / "index.json", "r") as f: + output_json = json.load(f) + assert "IANA — IANA-managed Reserved Domains" == output_json['history']['title'][0]['output'] + + with open(tmp_path / "index.html", "r") as f: + output_html = f.read() + assert "IANA — IANA-managed Reserved Domains" in output_html + diff --git a/tests/test_util.py b/tests/test_util.py new file mode 100644 index 00000000..19ed31c0 --- /dev/null +++ b/tests/test_util.py @@ -0,0 +1,21 @@ +#@enforce_types +#def download_url(url: str, timeout: int=None) -> str: +# """Download the contents of a remote url and return the text""" +# from .config import TIMEOUT, CHECK_SSL_VALIDITY, WGET_USER_AGENT +# timeout = timeout or TIMEOUT +# response = requests.get( +# url, +# headers={'User-Agent': WGET_USER_AGENT}, +# verify=CHECK_SSL_VALIDITY, +# timeout=timeout, +# ) +# if response.headers.get('Content-Type') == 'application/rss+xml': +# # Based on https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py +# _TEMPLATE = r'''%s\s*=\s*["']?\s*%s\s*["']?''' +# _XML_ENCODING_RE = _TEMPLATE % ('encoding', r'(?P[\w-]+)') +# _BODY_ENCODING_PATTERN = r'<\s*(\?xml\s[^>]+%s)' % (_XML_ENCODING_RE) +# _BODY_ENCODING_STR_RE = re.compile(_BODY_ENCODING_PATTERN, re.I | re.VERBOSE) +# match = _BODY_ENCODING_STR_RE.search(response.text[:1024]) +# if match: +# response.encoding = match.group('xmlcharset') +# return response.text \ No newline at end of file From 438203f4cec49e92c49976d57788be6b188f173e Mon Sep 17 00:00:00 2001 From: Cristian Date: Fri, 3 Jul 2020 12:54:21 -0500 Subject: [PATCH 2/4] test: add basic download_url test --- tests/test_util.py | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/tests/test_util.py b/tests/test_util.py index 19ed31c0..1497de5a 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -1,21 +1,5 @@ -#@enforce_types -#def download_url(url: str, timeout: int=None) -> str: -# """Download the contents of a remote url and return the text""" -# from .config import TIMEOUT, CHECK_SSL_VALIDITY, WGET_USER_AGENT -# timeout = timeout or TIMEOUT -# response = requests.get( -# url, -# headers={'User-Agent': WGET_USER_AGENT}, -# verify=CHECK_SSL_VALIDITY, -# timeout=timeout, -# ) -# if response.headers.get('Content-Type') == 'application/rss+xml': -# # Based on https://github.com/scrapy/w3lib/blob/master/w3lib/encoding.py -# _TEMPLATE = r'''%s\s*=\s*["']?\s*%s\s*["']?''' -# _XML_ENCODING_RE = _TEMPLATE % ('encoding', r'(?P[\w-]+)') -# _BODY_ENCODING_PATTERN = r'<\s*(\?xml\s[^>]+%s)' % (_XML_ENCODING_RE) -# _BODY_ENCODING_STR_RE = re.compile(_BODY_ENCODING_PATTERN, re.I | re.VERBOSE) -# match = _BODY_ENCODING_STR_RE.search(response.text[:1024]) -# if match: -# response.encoding = match.group('xmlcharset') -# return response.text \ No newline at end of file +from archivebox import util + +def test_download_url_downloads_content(): + text = util.download_url("https://example.com") + assert "Example Domain" in text \ No newline at end of file From 4302ae4caa4fccbe40e67084d4b3edd315e9eb1f Mon Sep 17 00:00:00 2001 From: Cristian Date: Fri, 3 Jul 2020 13:13:59 -0500 Subject: [PATCH 3/4] fix: Remove test section in setup.py --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 12002580..9ca39608 100755 --- a/setup.py +++ b/setup.py @@ -64,10 +64,8 @@ setuptools.setup( "sphinx", "sphinx-rtd-theme", "recommonmark", + "pytest", ], - "test": [ - "pytest" - ] # 'redis': ['redis', 'django-redis'], # 'pywb': ['pywb', 'redis'], }, From ffaae510779b49b44450c58c3c631a29f065ae32 Mon Sep 17 00:00:00 2001 From: apkallum Date: Fri, 3 Jul 2020 16:52:28 -0400 Subject: [PATCH 4/4] test github actions --- .github/workflows/test.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..311236c0 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,25 @@ +name: Test workflow +on: [push] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + with: + fetch-depth: 1 + + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + architecture: x64 + + - name: Install dependencies + run: | + pip install -e .[dev] + + - name: Test with pytest + run: | + pytest -s \ No newline at end of file