forks/archivebox
1
0
Fork 0
Code Releases Activity

new Dockerfile layout with better layering

Browse source
This commit is contained in:
Nick Sweeting 2023-10-20 02:47:34 -07:00
parent 86c662421b
commit 6736e63ef0
1 changed files with 103 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

171
Dockerfile
View file

@ -16,15 +16,17 @@
# Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development). # Archivebox](https://github.com/ArchiveBox/ArchiveBox#archivebox-development).
FROM python:3.11-slim-bullseye FROM debian:bookworm-backports
LABEL name="archivebox" \ LABEL name="archivebox" \
maintainer="Nick Sweeting <archivebox-docker@sweeting.me>" \ maintainer="Nick Sweeting <dockerfile@archivebox.io>" \
description="All-in-one personal internet archiving container" \ description="All-in-one personal internet archiving container" \
homepage="https://github.com/ArchiveBox/ArchiveBox" \ homepage="https://github.com/ArchiveBox/ArchiveBox" \
documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#docker" documentation="https://github.com/ArchiveBox/ArchiveBox/wiki/Docker#docker"
# System-level base config ######### Base System Setup ####################################
# Global system-level config
ENV TZ=UTC \ ENV TZ=UTC \
LANGUAGE=en_US:en \ LANGUAGE=en_US:en \
LC_ALL=C.UTF-8 \ LC_ALL=C.UTF-8 \
@ -32,103 +34,136 @@ ENV TZ=UTC \
PYTHONIOENCODING=UTF-8 \ PYTHONIOENCODING=UTF-8 \
PYTHONUNBUFFERED=1 \ PYTHONUNBUFFERED=1 \
DEBIAN_FRONTEND=noninteractive \ DEBIAN_FRONTEND=noninteractive \
APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
npm_config_loglevel=error
# Application-level base config # Application-level config
ENV CODE_DIR=/app \ ENV CODE_DIR=/app \
VENV_PATH=/venv \
DATA_DIR=/data \ DATA_DIR=/data \
NODE_DIR=/node \ GLOBAL_VENV=/venv \
APP_VENV=/app/.venv \
NODE_MODULES=/app/node_modules \
ARCHIVEBOX_USER="archivebox" ARCHIVEBOX_USER="archivebox"
ENV PATH="$PATH:$GLOBAL_VENV/bin:$APP_VENV/bin:$NODE_MODULES/.bin"
# Create non-privileged user for archivebox and chrome # Create non-privileged user for archivebox and chrome
RUN groupadd --system $ARCHIVEBOX_USER \ RUN groupadd --system $ARCHIVEBOX_USER \
&& useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER \
&& mkdir -p /etc/apt/keyrings
# Install system dependencies # Install system apt dependencies (adding backports to access more recent apt updates)
RUN apt-get update -qq \ RUN echo 'deb https://deb.debian.org/debian bullseye-backports main contrib non-free' >> /etc/apt/sources.list.d/backports.list \
&& apt-get install -qq -y --no-install-recommends \ && apt-get update -qq \
apt-transport-https ca-certificates gnupg2 zlib1g-dev \ && apt-get install -qq -y \
dumb-init gosu cron unzip curl \ apt-transport-https ca-certificates gnupg2 curl wget \
zlib1g-dev dumb-init gosu cron unzip \
nano iputils-ping dnsutils \
# 1. packaging dependencies
# 2. docker and init system dependencies
# 3. frivolous CLI helpers to make debugging failed archiving easier
&& mkdir -p /etc/apt/keyrings \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
######### Language Environments ####################################
# Install Node environment
RUN echo 'deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main' >> /etc/apt/sources.list.d/nodejs.list \
&& curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
&& apt-get update -qq \
&& apt-get install -qq -y nodejs \
&& npm i -g npm \
&& node --version \
&& npm --version
# Install Python environment
RUN apt-get update -qq \
&& apt-get install -qq -y -t bookworm-backports --no-install-recommends \
python3 python3-pip python3-venv python3-setuptools python3-wheel python-dev-is-python3 \
&& rm /usr/lib/python3*/EXTERNALLY-MANAGED \
&& python3 -m venv $GLOBAL_VENV \
&& $GLOBAL_VENV/bin/pip install --upgrade pip pdm setuptools wheel \
&& rm -rf /var/lib/apt/lists/*
######### Extractor Dependencies ##################################
# Install apt dependencies # Install apt dependencies
RUN apt-get update -qq \ RUN apt-get update -qq \
&& apt-get install -qq -y --no-install-recommends \ && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
wget curl chromium git ffmpeg youtube-dl ripgrep \ curl wget git yt-dlp ffmpeg ripgrep \
fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \ # Packages we have also needed in the past:
&& ln -s /usr/bin/chromium /usr/bin/chromium-browser \ # youtube-dl wget2 aria2 python3-pyxattr rtmpdump libfribidi-bin mpv \
# fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
# Install Node environment # Install chromium browser using playwright
RUN curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \ ENV PLAYWRIGHT_BROWSERS_PATH=/browsers
&& echo 'deb https://deb.nodesource.com/node_18.x buster main' >> /etc/apt/sources.list \ RUN apt-get update -qq \
&& apt-get update -qq \ && $GLOBAL_VENV/bin/pip install playwright \
&& apt-get install -qq -y --no-install-recommends \ && $GLOBAL_VENV/bin/playwright install --with-deps chromium \
nodejs \ && CHROME_BINARY="$($GLOBAL_VENV/bin/python -c 'from playwright.sync_api import sync_playwright; print(sync_playwright().start().chromium.executable_path)')" \
# && npm install -g npm \ && ln -s "$CHROME_BINARY" /usr/bin/chromium-browser \
&& rm -rf /var/lib/apt/lists/* && mkdir -p "/home/${ARCHIVEBOX_USER}/.config/chromium/Crash Reports/pending/" \
&& chown -R $ARCHIVEBOX_USER "/home/${ARCHIVEBOX_USER}/.config"
# Install Node dependencies # Install Node dependencies
WORKDIR "$NODE_DIR"
ENV PATH="${PATH}:$NODE_DIR/node_modules/.bin" \
npm_config_loglevel=error
ADD ./package.json ./package.json
ADD ./package-lock.json ./package-lock.json
RUN npm ci
# Install Python dependencies
WORKDIR "$CODE_DIR" WORKDIR "$CODE_DIR"
ENV PATH="${PATH}:$VENV_PATH/bin" ADD "package.json" "package-lock.json" "$CODE_DIR/"
RUN python -m venv --clear --symlinks "$VENV_PATH" \ RUN npm ci --prefer-offline --no-audit
&& pip install --upgrade --quiet pip setuptools \ RUN "$NODE_MODULES/.bin/readability-extractor" --version
&& mkdir -p "$CODE_DIR/archivebox"
ADD "./setup.py" "$CODE_DIR/" ######### Build Dependencies ####################################
ADD "./package.json" "$CODE_DIR/archivebox/"
WORKDIR "$CODE_DIR"
COPY --chown=root:root . "$CODE_DIR/"
# Install Python Build dependencies & build ArchiveBox package
# RUN apt-get update -qq \
# && apt-get install -qq -y -t bookworm-backports --no-install-recommends \
# build-essential libssl-dev libldap2-dev libsasl2-dev \
# && pdm venv create \
# && pdm install --fail-fast --no-lock --group :all \
# && pdm build \
# && apt-get purge -y \
# build-essential libssl-dev libldap2-dev libsasl2-dev \
# # these are only needed to build CPython libs, we discard after build phase to shrink layer size
# && apt-get autoremove -y \
# && rm -rf /var/lib/apt/lists/*
# Install ArchiveBox Python package from source
RUN apt-get update -qq \ RUN apt-get update -qq \
&& apt-get install -qq -y --no-install-recommends \ && $GLOBAL_VENV/bin/pip install -e "$CODE_DIR"[sonic,ldap]
build-essential python-dev python3-dev libldap2-dev libsasl2-dev \
&& echo 'empty placeholder for setup.py to use' > "$CODE_DIR/archivebox/README.md" \
&& python3 -c 'from distutils.core import run_setup; result = run_setup("./setup.py", stop_after="init"); print("\n".join(result.install_requires + result.extras_require["sonic"]))' > /tmp/requirements.txt \
&& pip install -r /tmp/requirements.txt \
&& pip install --upgrade youtube-dl yt-dlp \
&& apt-get purge -y build-essential python-dev python3-dev libldap2-dev libsasl2-dev \
&& apt-get autoremove -y \
&& rm -rf /var/lib/apt/lists/*
# Install apt development dependencies ####################################################
# RUN apt-get install -qq \
# && apt-get install -qq -y --no-install-recommends \
# python3 python3-dev python3-pip python3-venv python3-all \
# dh-python debhelper devscripts dput software-properties-common \
# python3-distutils python3-setuptools python3-wheel python3-stdeb
# RUN python3 -c 'from distutils.core import run_setup; result = run_setup("./setup.py", stop_after="init"); print("\n".join(result.extras_require["dev"]))' > /tmp/dev_requirements.txt \
# && pip install --quiet -r /tmp/dev_requirements.txt
# Install ArchiveBox Python package and its dependencies
WORKDIR "$CODE_DIR"
ADD . "$CODE_DIR"
RUN chown -R root:root . && chmod a+rX -R . && pip install -e .
# Setup ArchiveBox runtime config # Setup ArchiveBox runtime config
WORKDIR "$DATA_DIR"
ENV IN_DOCKER=True \ ENV IN_DOCKER=True \
WGET_BINARY="wget" \
YOUTUBEDL_BINARY="yt-dlp" \
CHROME_SANDBOX=False \ CHROME_SANDBOX=False \
CHROME_BINARY="/usr/bin/chromium-browser" \ CHROME_BINARY="/usr/bin/chromium-browser" \
USE_SINGLEFILE=True \ USE_SINGLEFILE=True \
SINGLEFILE_BINARY="$NODE_DIR/node_modules/.bin/single-file" \ SINGLEFILE_BINARY="$NODE_MODULES/.bin/single-file" \
USE_READABILITY=True \ USE_READABILITY=True \
READABILITY_BINARY="$NODE_DIR/node_modules/.bin/readability-extractor" \ READABILITY_BINARY="$NODE_MODULES/.bin/readability-extractor" \
USE_MERCURY=True \ USE_MERCURY=True \
MERCURY_BINARY="$NODE_DIR/node_modules/.bin/mercury-parser" \ MERCURY_BINARY="$NODE_MODULES/.bin/postlight-parser"
YOUTUBEDL_BINARY="yt-dlp"
# Print version for nice docker finish summary # Print version for nice docker finish summary
# RUN archivebox version # RUN archivebox version
RUN /app/bin/docker_entrypoint.sh archivebox version RUN echo "[√] Finished Docker build succesfully. Saving build summary in: /version_info.txt" \
&& uname -a | tee -a /version_info.txt \
&& env --chdir="$NODE_DIR" npm version | tee -a /version_info.txt \
&& env --chdir="$CODE_DIR" pdm info | tee -a /version_info.txt \
&& "$CODE_DIR/bin/docker_entrypoint.sh" archivebox version 2>&1 | tee -a /version_info.txt
####################################################
# Open up the interfaces to the outside world # Open up the interfaces to the outside world
VOLUME "$DATA_DIR" VOLUME "/data"
EXPOSE 8000 EXPOSE 8000
# Optional: # Optional: