forks/archivebox
1
0
Fork 0
Code Releases Activity

revert back to python-3.11 base image and add cache dirs for massive build speed increase

Browse source
This commit is contained in:
Nick Sweeting 2023-10-31 03:58:24 -07:00
parent ce4df6d965
commit 562fd5a3bb
3 changed files with 36 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
Dockerfile
View file

@ -16,7 +16,7 @@
# Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/ # Use Debian 12 w/ faster package updates: https://packages.debian.org/bookworm-backports/
FROM debian:bookworm-backports FROM python:3.11-slim-bookworm
LABEL name="archivebox" \ LABEL name="archivebox" \
maintainer="Nick Sweeting <dockerfile@archivebox.io>" \ maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
@ -98,8 +98,9 @@ RUN echo "[*] Setting up $ARCHIVEBOX_USER user uid=${DEFAULT_PUID}..." \
# https://docs.linuxserver.io/general/understanding-puid-and-pgid # https://docs.linuxserver.io/general/understanding-puid-and-pgid
# Install system apt dependencies (adding backports to access more recent apt updates) # Install system apt dependencies (adding backports to access more recent apt updates)
RUN echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \ RUN --mount=type=cache,target=/var/cache/apt \
# && echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' >> /etc/apt/sources.list.d/backports.list \ echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
&& echo 'deb https://deb.debian.org/debian bookworm-backports main contrib non-free' >> /etc/apt/sources.list.d/backports.list \
&& mkdir -p /etc/apt/keyrings \ && mkdir -p /etc/apt/keyrings \
&& apt-get update -qq \ && apt-get update -qq \
&& apt-get install -qq -y -t bookworm-backports --no-install-recommends \ && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
@ -114,7 +115,8 @@ RUN echo "[+] Installing APT base system dependencies for $TARGETPLATFORM..." \
######### Language Environments #################################### ######### Language Environments ####################################
# Install Node environment # Install Node environment
RUN echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \ RUN --mount=type=cache,target=/var/cache/apt \
echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
&& echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \ && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_VERSION}.x nodistro main" >> /etc/apt/sources.list.d/nodejs.list \
&& curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \ && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
&& apt-get update -qq \ && apt-get update -qq \
@ -131,30 +133,34 @@ RUN echo "[+] Installing Node $NODE_VERSION environment in $NODE_MODULES..." \
) | tee -a /VERSION.txt ) | tee -a /VERSION.txt
# Install Python environment # Install Python environment
RUN echo "[+] Installing Python $PYTHON_VERSION environment in $GLOBAL_VENV and $APP_VENV..." \ RUN --mount=type=cache,target=/root/.cache/pip --mount=type=cache,target=/var/cache/apt \
echo "[+] Installing Python $PYTHON_VERSION environment in $GLOBAL_VENV and $APP_VENV..." \
&& apt-get update -qq \ && apt-get update -qq \
&& apt-get install -qq -y -t bookworm-backports --no-install-recommends \ && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
python3 python3-pip python3-setuptools python3-wheel python3-venv python3-pdm python-dev-is-python3 \ python3 python3-pip python3-setuptools python3-wheel python3-venv python-dev-is-python3 \
#python3-ldap libldap2-dev libsasl2-dev libssl-dev python3-msgpack \ #python3-ldap libldap2-dev libsasl2-dev libssl-dev python3-msgpack \
&& rm -rf /var/lib/apt/lists/* \ && rm -rf /var/lib/apt/lists/* \
# tell PDM to allow using global system python site packages # tell PDM to allow using global system python site packages
&& rm /usr/lib/python3*/EXTERNALLY-MANAGED \ # && rm /usr/lib/python3*/EXTERNALLY-MANAGED \
# create global virtual environment GLOBAL_VENV to use (better than using pip install --global) # create global virtual environment GLOBAL_VENV to use (better than using pip install --global)
&& python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \ # && python3 -m venv --system-site-packages --symlinks $GLOBAL_VENV \
# && python3 -m venv --system-site-packages $GLOBAL_VENV \
# && python3 -m venv $GLOBAL_VENV \
# install global dependencies / python build dependencies in GLOBAL_VENV # install global dependencies / python build dependencies in GLOBAL_VENV
&& $GLOBAL_VENV/bin/pip install --break-system-packages --upgrade pip pdm setuptools wheel \ && pip install --upgrade pip setuptools wheel \
# Save version info # Save version info
&& ( \ && ( \
which python3 && python3 --version | grep " $PYTHON_VERSION" \ which python3 && python3 --version | grep " $PYTHON_VERSION" \
&& which pip3 && pip3 --version \ && which pip && pip --version \
&& which pdm && pdm --version \ # && which pdm && pdm --version \
&& echo -e '\n\n' \ && echo -e '\n\n' \
) | tee -a /VERSION.txt ) | tee -a /VERSION.txt
######### Extractor Dependencies ################################## ######### Extractor Dependencies ##################################
# Install apt dependencies # Install apt dependencies
RUN echo "[+] Installing APT extractor dependencies globally using apt..." \ RUN --mount=type=cache,target=/var/cache/apt \
echo "[+] Installing APT extractor dependencies globally using apt..." \
&& apt-get update -qq \ && apt-get update -qq \
&& apt-get install -qq -y -t bookworm-backports --no-install-recommends \ && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
curl wget git yt-dlp ffmpeg ripgrep \ curl wget git yt-dlp ffmpeg ripgrep \
@ -173,11 +179,12 @@ RUN echo "[+] Installing APT extractor dependencies globally using apt..." \
) | tee -a /VERSION.txt ) | tee -a /VERSION.txt
# Install chromium browser using playwright # Install chromium browser using playwright
RUN echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \ RUN --mount=type=cache,target=/var/cache/apt \
echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PATH..." \
&& apt-get update -qq \ && apt-get update -qq \
&& if [[ "$TARGETPLATFORM" == "linux/amd64" || "$TARGETPLATFORM" == "linux/arm64" ]]; then \ && if [[ "$TARGETPLATFORM" == "linux/amd64" || "$TARGETPLATFORM" == "linux/arm64" ]]; then \
# install Chromium using playwright # install Chromium using playwright
$GLOBAL_VENV/bin/pip install playwright \ pip install playwright \
&& $GLOBAL_VENV/bin/playwright install --with-deps chromium \ && $GLOBAL_VENV/bin/playwright install --with-deps chromium \
&& export CHROME_BINARY="$($GLOBAL_VENV/bin/python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')"; \ && export CHROME_BINARY="$($GLOBAL_VENV/bin/python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')"; \
else \ else \
@ -199,8 +206,9 @@ RUN echo "[+] Installing Browser binary dependencies to $PLAYWRIGHT_BROWSERS_PAT
# Install Node dependencies # Install Node dependencies
WORKDIR "$CODE_DIR" WORKDIR "$CODE_DIR"
COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR/" COPY --chown=root:root --chmod=755 "package.json" "package-lock.json" "$CODE_DIR/"
RUN echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \ RUN --mount=type=cache,target=/root/.npm \
&& npm ci --prefer-offline --no-audit \ echo "[+] Installing NPM extractor dependencies from package.json into $NODE_MODULES..." \
&& npm ci --prefer-offline --no-audit --cache /root/.npm \
&& ( \ && ( \
which node && node --version \ which node && node --version \
&& which npm && npm version \ && which npm && npm version \
@ -211,18 +219,18 @@ RUN echo "[+] Installing NPM extractor dependencies from package.json into $NODE
# Install ArchiveBox Python dependencies # Install ArchiveBox Python dependencies
WORKDIR "$CODE_DIR" WORKDIR "$CODE_DIR"
COPY --chown=root:root --chmod=755 "./pyproject.toml" "./pdm.lock" "requirements.txt" "$CODE_DIR/" COPY --chown=root:root --chmod=755 "./pyproject.toml" "requirements.txt" "$CODE_DIR/"
RUN --mount=type=cache,target=/root/.cache/pip \ RUN --mount=type=cache,target=/root/.cache/pip --mount=type=cache,target=/var/cache/apt \
&& echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt into $GLOBAL_VENV..." \ echo "[+] Installing PIP ArchiveBox dependencies from requirements.txt into $GLOBAL_VENV..." \
&& apt-get update -qq \ && apt-get update -qq \
&& apt-get install -qq -y -t bookworm-backports --no-install-recommends \ && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
build-essential libssl-dev libldap2-dev libsasl2-dev \ build-essential libssl-dev libldap2-dev libsasl2-dev \
&& ln -s "$GLOBAL_VENV" "$APP_VENV" \ # && ln -s "$GLOBAL_VENV" "$APP_VENV" \
# && pdm use --venv in-project \ # && pdm use --venv in-project \
# && pdm run python -m ensurepip \ # && pdm run python -m ensurepip \
# && pdm sync --fail-fast --no-editable --group :all --no-self \ # && pdm sync --fail-fast --no-editable --group :all --no-self \
# && pdm export -o requirements.txt --without-hashes \ # && pdm export -o requirements.txt --without-hashes \
&& source $GLOBAL_VENV/bin/activate \ # && source $GLOBAL_VENV/bin/activate \
&& pip install -r requirements.txt \ && pip install -r requirements.txt \
&& apt-get purge -y \ && apt-get purge -y \
build-essential libssl-dev libldap2-dev libsasl2-dev \ build-essential libssl-dev libldap2-dev libsasl2-dev \
@ -232,7 +240,8 @@ RUN --mount=type=cache,target=/root/.cache/pip \
# Install ArchiveBox Python package from source # Install ArchiveBox Python package from source
COPY --chown=root:root --chmod=755 "." "$CODE_DIR/" COPY --chown=root:root --chmod=755 "." "$CODE_DIR/"
RUN echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \ RUN --mount=type=cache,target=/root/.cache/pip --mount=type=cache,target=/var/cache/apt \
echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
&& apt-get update -qq \ && apt-get update -qq \
# install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi # install C compiler to build deps on platforms that dont have 32-bit wheels available on pypi
&& if [[ "$TARGETPLATFORM" == "linux/arm/v7" ]]; then \ && if [[ "$TARGETPLATFORM" == "linux/arm/v7" ]]; then \
@ -243,7 +252,7 @@ RUN echo "[*] Installing PIP ArchiveBox package from $CODE_DIR..." \
procps; \ procps; \
fi \ fi \
# INSTALL ARCHIVEBOX python package globally from CODE_DIR, with all optional dependencies # INSTALL ARCHIVEBOX python package globally from CODE_DIR, with all optional dependencies
&& $GLOBAL_VENV/bin/pip3 install --break-system-packages -e "$CODE_DIR"[sonic,ldap] \ && pip install --break-system-packages -e "$CODE_DIR"[sonic,ldap] \
# save docker image size and always remove compilers / build tools after building is complete # save docker image size and always remove compilers / build tools after building is complete
&& apt-get purge -y build-essential \ && apt-get purge -y build-essential \
&& apt-get autoremove -y \ && apt-get autoremove -y \

6
pdm.lock
View file

@ -443,7 +443,7 @@ files = [
[[package]] [[package]]
name = "ipython" name = "ipython"
version = "8.17.1" version = "8.17.2"
requires_python = ">=3.9" requires_python = ">=3.9"
summary = "IPython: Productive Interactive Computing" summary = "IPython: Productive Interactive Computing"
dependencies = [ dependencies = [
@ -461,8 +461,8 @@ dependencies = [
"typing-extensions; python_version < \"3.10\"", "typing-extensions; python_version < \"3.10\"",
] ]
files = [ files = [
{file = "ipython-8.17.1-py3-none-any.whl", hash = "sha256:b4510d0a163c89c78cf81be68acc841dde4a8290b6ee3f3c9578ccdd2574553c"}, {file = "ipython-8.17.2-py3-none-any.whl", hash = "sha256:1e4d1d666a023e3c93585ba0d8e962867f7a111af322efff6b9c58062b3e5444"},
{file = "ipython-8.17.1.tar.gz", hash = "sha256:9e12020f4bf74f08631c0f033bb580e074fbe36c64903195f3e63b9c0a986cbe"}, {file = "ipython-8.17.2.tar.gz", hash = "sha256:126bb57e1895594bb0d91ea3090bbd39384f6fe87c3d57fd558d0670f50339bb"},
] ]
[[package]] [[package]]

2
requirements.txt
View file

@ -16,7 +16,7 @@ django-extensions==3.1.5
executing==2.0.1 executing==2.0.1
greenlet==3.0.0 greenlet==3.0.0
idna==3.4 idna==3.4
ipython==8.17.1 ipython==8.17.2
jedi==0.19.1 jedi==0.19.1
matplotlib-inline==0.1.6 matplotlib-inline==0.1.6
mutagen==1.47.0 mutagen==1.47.0