use setup.py to determine dependencies in Dockerfile instead of egg-info requires.txt

Dockerfile

@@ -79,13 +79,13 @@ WORKDIR "$CODE_DIR"
 ENV PATH="${PATH}:$VENV_PATH/bin"
 RUN python -m venv --clear --symlinks "$VENV_PATH" \
     && pip install --upgrade --quiet pip setuptools
-ADD ./pip_dist/archivebox.egg-info/requires.txt "$CODE_DIR/pip_dist/archivebox.egg-info/requires.txt"
+ADD "./setup.py" "$CODE_DIR/"
+ADD "./README.md" "./package.json" "$CODE_DIR/archivebox/"
 RUN apt-get update -qq \
     && apt-get install -qq -y --no-install-recommends \
         build-essential python-dev python3-dev \
-    # && pip install --upgrade pip \
+    && python3 -c 'from distutils.core import run_setup; result = run_setup("./setup.py", stop_after="init"); print("\n".join(result.install_requires + result.extras_require["sonic"]))' > /tmp/requirements.txt \
-    && grep -B 1000 -E '^$' "$CODE_DIR/pip_dist/archivebox.egg-info/requires.txt" | pip install --quiet -r /dev/stdin \
+    && pip install --quiet -r /tmp/requirements.txt \
-    && pip install --quiet "sonic-client==0.0.5" \
     && apt-get purge -y build-essential python-dev python3-dev \
     && apt-get autoremove -y \
     && rm -rf /var/lib/apt/lists/*

archivebox/config.py

@@ -1076,6 +1076,11 @@ def setup_django(out_dir: Path=None, check_db=False, config: ConfigDict=CONFIG,
         else:
             django.setup()
             
+            # Enable WAL mode in sqlite3
+            from django.db import connection
+            with connection.cursor() as cursor:
+                cursor.execute("PRAGMA journal_mode=wal;")
+
         if check_db:
             sql_index_path = Path(output_dir) / SQL_INDEX_FILENAME
             assert sql_index_path.exists(), (

archivebox/core/settings.py

@@ -107,9 +107,6 @@ DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
         'NAME': DATABASE_NAME,
-        'OPTIONS': {
-            'init_command': 'PRAGMA journal_mode=wal;',
-        }
     }
 }
 

setup.py

@@ -27,52 +27,30 @@ PACKAGE_DIR = ROOT_DIR / PKG_NAME
 README = (PACKAGE_DIR / "README.md").read_text(encoding='utf-8', errors='ignore')
 VERSION = json.loads((PACKAGE_DIR / "package.json").read_text().strip())['version']
 
-# To see when setup.py gets called (uncomment for debugging):
+PYTHON_REQUIRES = ">=3.7"
-# import sys
+SETUP_REQUIRES = ["wheel"]
-# print(PACKAGE_DIR, f"     (v{VERSION})")
+INSTALL_REQUIRES = [
-# print('>', sys.executable, *sys.argv)
-
-
-class DisabledTestCommand(test):
-    def run(self):
-        # setup.py test is deprecated, disable it here by force so stdeb doesnt run it
-        print('Use the ./bin/test.sh script to run tests, not setup.py test.')
-
-
-setuptools.setup(
-    name=PKG_NAME,
-    version=VERSION,
-    license=LICENSE,
-    author=AUTHOR,
-    author_email=AUTHOR_EMAIL,
-    description=DESCRIPTION,
-    long_description=README,
-    long_description_content_type="text/markdown",
-    url=REPO_URL,
-    project_urls=PROJECT_URLS,
-    python_requires=">=3.7",
-    setup_requires=[
-        "wheel",
-    ],
-    install_requires=[
     # only add things here that have corresponding apt python3-packages available
     # anything added here also needs to be added to our package dependencies in
     # stdeb.cfg (apt), archivebox.rb (brew), Dockerfile, etc.
     # if there is no apt python3-package equivalent, then vendor it instead in
     # ./archivebox/vendor/
-        "requests==2.24.0",
+    "requests>=2.24.0",
-        "atomicwrites==1.4.0",
+    "atomicwrites>=1.4.0",
-        "mypy-extensions==0.4.3",
+    "mypy-extensions>=0.4.3",
-        "django==3.1.3",
+    "django>=3.1.3",
-        "django-extensions==3.0.3",
+    "django-extensions>=3.0.3",
     "dateparser",
     "ipython",
     "youtube-dl",
-        "python-crontab==2.5.1",
+    "python-crontab>=2.5.1",
-        "croniter==0.3.34",
+    "croniter>=0.3.34",
-        "w3lib==1.22.0",
+    "w3lib>=1.22.0",
+]
+EXTRAS_REQUIRE = {
+    'sonic': [
+        "sonic-client>=0.0.5",
     ],
-    extras_require={
     'dev': [
         "setuptools",
         "twine",
@@ -88,7 +66,37 @@ setuptools.setup(
         "bottle",
         "stdeb",
     ],
-    },
+}
+
+# To see when setup.py gets called (uncomment for debugging):
+# import sys
+# print(PACKAGE_DIR, f"     (v{VERSION})")
+# print('>', sys.executable, *sys.argv)
+
+
+class DisabledTestCommand(test):
+    def run(self):
+        # setup.py test is deprecated, disable it here by force so stdeb doesnt run it
+        print()
+        print('[X] Running tests via setup.py test is deprecated.')
+        print('    Hint: Use the ./bin/test.sh script or pytest instead')
+
+
+setuptools.setup(
+    name=PKG_NAME,
+    version=VERSION,
+    license=LICENSE,
+    author=AUTHOR,
+    author_email=AUTHOR_EMAIL,
+    description=DESCRIPTION,
+    long_description=README,
+    long_description_content_type="text/markdown",
+    url=REPO_URL,
+    project_urls=PROJECT_URLS,
+    python_requires=PYTHON_REQUIRES,
+    setup_requires=SETUP_REQUIRES,
+    install_requires=INSTALL_REQUIRES,
+    extras_require=EXTRAS_REQUIRE,
     packages=[PKG_NAME],
     include_package_data=True,   # see MANIFEST.in
     entry_points={