From e1d0b8bce73c8a919da9915e99a5c51089b0a138 Mon Sep 17 00:00:00 2001 From: Cristian Date: Mon, 26 Oct 2020 07:45:21 -0500 Subject: [PATCH 1/3] feat: Initialize django at the beginning --- archivebox/__init__.py | 4 ++++ archivebox/index/sql.py | 9 +-------- archivebox/main.py | 5 ----- tests/fixtures.py | 1 + tests/test_add.py | 1 + 5 files changed, 7 insertions(+), 13 deletions(-) diff --git a/archivebox/__init__.py b/archivebox/__init__.py index b0c00b61..d41776ba 100644 --- a/archivebox/__init__.py +++ b/archivebox/__init__.py @@ -1 +1,5 @@ __package__ = 'archivebox' +from .config import setup_django, OUTPUT_DIR + +print(OUTPUT_DIR) +setup_django() diff --git a/archivebox/index/sql.py b/archivebox/index/sql.py index aa7c8817..42d29b66 100644 --- a/archivebox/index/sql.py +++ b/archivebox/index/sql.py @@ -7,14 +7,13 @@ from django.db.models import QuerySet from .schema import Link from ..util import enforce_types -from ..config import setup_django, OUTPUT_DIR +from ..config import OUTPUT_DIR ### Main Links Index @enforce_types def parse_sql_main_index(out_dir: Path=OUTPUT_DIR) -> Iterator[Link]: - setup_django(out_dir, check_db=True) from core.models import Snapshot return ( @@ -24,7 +23,6 @@ def parse_sql_main_index(out_dir: Path=OUTPUT_DIR) -> Iterator[Link]: @enforce_types def remove_from_sql_main_index(snapshots: QuerySet, out_dir: Path=OUTPUT_DIR) -> None: - setup_django(out_dir, check_db=True) from django.db import transaction with transaction.atomic(): @@ -51,7 +49,6 @@ def write_link_to_sql_index(link: Link): @enforce_types def write_sql_main_index(links: List[Link], out_dir: Path=OUTPUT_DIR) -> None: - setup_django(out_dir, check_db=True) from django.db import transaction with transaction.atomic(): @@ -61,7 +58,6 @@ def write_sql_main_index(links: List[Link], out_dir: Path=OUTPUT_DIR) -> None: @enforce_types def write_sql_link_details(link: Link, out_dir: Path=OUTPUT_DIR) -> None: - setup_django(out_dir, check_db=True) from core.models import Snapshot from django.db import transaction @@ -84,7 +80,6 @@ def write_sql_link_details(link: Link, out_dir: Path=OUTPUT_DIR) -> None: @enforce_types def list_migrations(out_dir: Path=OUTPUT_DIR) -> List[Tuple[bool, str]]: - setup_django(out_dir, check_db=False) from django.core.management import call_command out = StringIO() call_command("showmigrations", list=True, stdout=out) @@ -101,7 +96,6 @@ def list_migrations(out_dir: Path=OUTPUT_DIR) -> List[Tuple[bool, str]]: @enforce_types def apply_migrations(out_dir: Path=OUTPUT_DIR) -> List[str]: - setup_django(out_dir, check_db=False) from django.core.management import call_command null, out = StringIO(), StringIO() call_command("makemigrations", interactive=False, stdout=null) @@ -112,6 +106,5 @@ def apply_migrations(out_dir: Path=OUTPUT_DIR) -> List[str]: @enforce_types def get_admins(out_dir: Path=OUTPUT_DIR) -> List[str]: - setup_django(out_dir, check_db=False) from django.contrib.auth.models import User return User.objects.filter(is_superuser=True) diff --git a/archivebox/main.py b/archivebox/main.py index 72d5009c..8e695a56 100644 --- a/archivebox/main.py +++ b/archivebox/main.py @@ -82,7 +82,6 @@ from .config import ( check_dependencies, check_data_folder, write_config_file, - setup_django, VERSION, CODE_LOCATIONS, EXTERNAL_LOCATIONS, @@ -305,7 +304,6 @@ def init(force: bool=False, out_dir: Path=OUTPUT_DIR) -> None: else: print('\n{green}[+] Building main SQL index and running migrations...{reset}'.format(**ANSI)) - setup_django(out_dir, check_db=False) DATABASE_FILE = Path(out_dir) / SQL_INDEX_FILENAME print(f' √ {DATABASE_FILE}') print() @@ -1033,7 +1031,6 @@ def server(runserver_args: Optional[List[str]]=None, config.DEBUG = config.DEBUG or debug check_data_folder(out_dir=out_dir) - setup_django(out_dir) from django.core.management import call_command from django.contrib.auth.models import User @@ -1070,7 +1067,6 @@ def manage(args: Optional[List[str]]=None, out_dir: Path=OUTPUT_DIR) -> None: """Run an ArchiveBox Django management command""" check_data_folder(out_dir=out_dir) - setup_django(out_dir) from django.core.management import execute_from_command_line if (args and "createsuperuser" in args) and (IN_DOCKER and not IS_TTY): @@ -1087,7 +1083,6 @@ def shell(out_dir: Path=OUTPUT_DIR) -> None: check_data_folder(out_dir=out_dir) - setup_django(OUTPUT_DIR) from django.core.management import call_command call_command("shell_plus") diff --git a/tests/fixtures.py b/tests/fixtures.py index 6dd4cb28..fbff5e0f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -6,6 +6,7 @@ import pytest @pytest.fixture def process(tmp_path): os.chdir(tmp_path) + print("should be at", tmp_path) process = subprocess.run(['archivebox', 'init'], capture_output=True) return process diff --git a/tests/test_add.py b/tests/test_add.py index 5e672e8d..37b13653 100644 --- a/tests/test_add.py +++ b/tests/test_add.py @@ -32,6 +32,7 @@ def test_depth_flag_0_crawls_only_the_arg_page(tmp_path, process, disable_extrac env=disable_extractors_dict, ) + breakpoint() archived_item_path = list(tmp_path.glob('archive/**/*'))[0] with open(archived_item_path / "index.json", "r") as f: output_json = json.load(f) From a6bee5f11100f0fb0901205db5fb646b46777b22 Mon Sep 17 00:00:00 2001 From: Cristian Date: Mon, 26 Oct 2020 08:02:04 -0500 Subject: [PATCH 2/3] feat: Move setup_django to an inner module --- archivebox/__init__.py | 4 ---- archivebox/cli/__init__.py | 4 ++++ tests/test_add.py | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/archivebox/__init__.py b/archivebox/__init__.py index d41776ba..b0c00b61 100644 --- a/archivebox/__init__.py +++ b/archivebox/__init__.py @@ -1,5 +1 @@ __package__ = 'archivebox' -from .config import setup_django, OUTPUT_DIR - -print(OUTPUT_DIR) -setup_django() diff --git a/archivebox/cli/__init__.py b/archivebox/cli/__init__.py index 83055e8e..f41ba5dc 100644 --- a/archivebox/cli/__init__.py +++ b/archivebox/cli/__init__.py @@ -134,3 +134,7 @@ __all__ = ( 'run_subcommand', *SUBCOMMANDS.keys(), ) + + +from ..config import setup_django +setup_django() diff --git a/tests/test_add.py b/tests/test_add.py index 37b13653..5e672e8d 100644 --- a/tests/test_add.py +++ b/tests/test_add.py @@ -32,7 +32,6 @@ def test_depth_flag_0_crawls_only_the_arg_page(tmp_path, process, disable_extrac env=disable_extractors_dict, ) - breakpoint() archived_item_path = list(tmp_path.glob('archive/**/*'))[0] with open(archived_item_path / "index.json", "r") as f: output_json = json.load(f) From f6ce1de8827367997dea32ec6cd691803d3acec5 Mon Sep 17 00:00:00 2001 From: Cristian Date: Tue, 27 Oct 2020 09:11:41 -0500 Subject: [PATCH 3/3] fix: archivebox version was being called as root --- Dockerfile | 3 ++- tests/fixtures.py | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5f16e658..d67220a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -99,7 +99,8 @@ ENV IN_DOCKER=True \ MERCURY_BINARY="$NODE_DIR/node_modules/.bin/mercury-parser" # Print version for nice docker finish summary -RUN archivebox version +# RUN archivebox version +RUN /app/bin/docker_entrypoint.sh archivebox version # Open up the interfaces to the outside world VOLUME "$DATA_DIR" diff --git a/tests/fixtures.py b/tests/fixtures.py index fbff5e0f..cca722f3 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -6,7 +6,6 @@ import pytest @pytest.fixture def process(tmp_path): os.chdir(tmp_path) - print("should be at", tmp_path) process = subprocess.run(['archivebox', 'init'], capture_output=True) return process @@ -26,4 +25,4 @@ def disable_extractors_dict(): "SAVE_MEDIA": "false", "SAVE_ARCHIVE_DOT_ORG": "false" }) - return env \ No newline at end of file + return env