install node dependencies from package.json

.dockerignore

@@ -8,7 +8,6 @@ venv/
 .venv/
 .docker-venv/
 
-*.egg-info/
 build/
 dist/
 

.github/workflows/test.yml

@@ -40,28 +40,19 @@ jobs:
         with:
           fetch-depth: 1
 
-      - uses: actions/checkout@v2
+      ### Setup Python & JS Languages
-        with:
-          fetch-depth: 1
-          repository: "gildas-lormeau/SingleFile"
-          ref: "master"
-          path: "singlefile"
-
-      - name: Install npm requirements for singlefile
-        run: npm install --prefix singlefile/cli
-
-      - name: Give singlefile execution permissions
-        run: chmod +x singlefile/cli/single-file
-
-      - name: Set SINGLEFILE_BINARY
-        run: echo "::set-env name=SINGLEFILE_BINARY::$GITHUB_WORKSPACE/singlefile/cli/single-file"
-
       - name: Set up Python ${{ matrix.python }}
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python }}
           architecture: x64
 
+      - name: Set up Node JS 14.7.0
+        uses: actions/setup-node@v1
+        with:
+          node-version: 14.7.0
+
+      ### Install Python & JS Dependencies
       - name: Get pip cache dir
         id: pip-cache
         run: |
@@ -76,19 +67,39 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-${{ matrix.python }}-venv-
 
-      - name: Use nodejs 14.7.0
+       - name: Install pip dependencies
-        uses: actions/setup-node@v1
-        with:
-          node-version: 14.7.0
-
-      - name: Debug
-        run: ls ./
-
-      - name: Install dependencies
         run: |
           python -m pip install .
           python -m pip install pytest bottle
 
+
+      - name: Get npm cache dir
+        id: npm-cache
+        run: |
+          echo "::set-output name=dir::$GITHUB_WORKSPACE/node_modules"
+
+      - name: Cache npm
+        uses: actions/cache@v2
+        id: cache-npm
+        with:
+          path: ${{ steps.npm-cache.outputs.dir }}
+          key: ${{ runner.os }}-node_modules-${{ hashFiles('package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node_modules
+
+      - name: Install npm requirements
+        run: |
+          npm install
+          echo "::set-env name=SINGLEFILE_BINARY::$GITHUB_WORKSPACE/node_modules/.bin/single-file"
+          echo "::set-env name=READABILITY_BINARY::$GITHUB_WORKSPACE/node_modules/.bin/readability-extractor"
+
+
+      ### Run the tests
+      - name: Directory listing for debugging
+        run: |
+          pwd
+          ls -a ./
+
       - name: Test built package with pytest
         run: |
           python -m pytest -s

Dockerfile

@@ -1,81 +1,105 @@
-# This is the Dockerfile for ArchiveBox, it includes the following major pieces:
+# This is the Dockerfile for ArchiveBox, it bundles the following dependencies:
-#     git, curl, wget, python3, youtube-dl, google-chrome-stable, ArchiveBox
+#     python3, ArchiveBox, curl, wget, git, chromium, youtube-dl, single-file
 # Usage:
-#     docker build . -t archivebox
+#     docker build . -t archivebox --no-cache
 #     docker run -v "$PWD/data":/data archivebox init
 #     docker run -v "$PWD/data":/data archivebox add 'https://example.com'
-# Documentation:
+#     docker run -v "$PWD/data":/data -it archivebox manage createsuperuser
-#     https://github.com/pirate/ArchiveBox/wiki/Docker#docker
+#     docker run -v "$PWD/data":/data -p 8000:8000 archivebox server
 
 FROM python:3.8-slim-buster
 
 LABEL name="archivebox" \
-    maintainer="Nick Sweeting <archivebox-git@sweeting.me>" \
+    maintainer="Nick Sweeting <archivebox-docker@sweeting.me>" \
-    description="All-in-one personal internet archiving container"
+    description="All-in-one personal internet archiving container" \
+    homepage="https://github.com/pirate/ArchiveBox" \
+    documentation="https://github.com/pirate/ArchiveBox/wiki/Docker#docker"
 
+# System-level base config
 ENV TZ=UTC \
     LANGUAGE=en_US:en \
     LC_ALL=C.UTF-8 \
     LANG=C.UTF-8 \
     PYTHONIOENCODING=UTF-8 \
     PYTHONUNBUFFERED=1 \
-    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 \
+    DEBIAN_FRONTEND=noninteractive \
-    CODE_PATH=/app \
+    APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1
+
+# Application-level base config
+ENV CODE_DIR=/app \
     VENV_PATH=/venv \
-    DATA_PATH=/data \
+    DATA_DIR=/data \
-    EXTRA_PATH=/extra
+    NODE_DIR=/node \
+    ARCHIVEBOX_USER="archivebox"
 
-# First install CLI utils and base deps, then Chrome + Fons + nodejs
+# Create non-privileged user for archivebox and chrome
-RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections \
+RUN groupadd --system $ARCHIVEBOX_USER \
+    && useradd --system --create-home --gid $ARCHIVEBOX_USER --groups audio,video $ARCHIVEBOX_USER
+
+# Install system dependencies
+RUN apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+        apt-transport-https ca-certificates apt-utils gnupg gnupg2 libgconf-2-4 zlib1g-dev \
+        dumb-init gosu unzip curl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Python environment
+RUN apt-get update -qq \
+    && apt-get install -qq -y --no-install-recommends \
+        ipython3 python-dev python3-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Node environment
+RUN curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - \
+    && echo 'deb https://deb.nodesource.com/node_14.x buster main' >> /etc/apt/sources.list \
     && apt-get update -qq \
     && apt-get install -qq -y --no-install-recommends \
-    apt-transport-https ca-certificates apt-utils gnupg gosu gnupg2 libgconf-2-4 zlib1g-dev \
+        nodejs \
-    dumb-init jq git wget curl youtube-dl ffmpeg \
+    && rm -rf /var/lib/apt/lists/*
-    && curl -sL https://deb.nodesource.com/setup_14.x | bash - \
+
-    && apt-get update -qq \
+# Install apt dependencies
+RUN apt-get update -qq \
     && apt-get install -qq -y --no-install-recommends \
-    chromium \
+        wget curl chromium git ffmpeg youtube-dl \
-    fontconfig \
+        fontconfig fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-symbola fonts-noto fonts-freefont-ttf \
-    fonts-ipafont-gothic \
+    && rm -rf /var/lib/apt/lists/*
-    fonts-wqy-zenhei \
-    fonts-thai-tlwg \
-    fonts-kacst \
-    fonts-symbola \
-    fonts-noto \
-    fonts-freefont-ttf \
-    nodejs \
-    unzip \
-    python3 python3-pip ipython3 build-essential python-dev python3-dev \
-    && rm -rf /var/lib/apt/lists/* 
 
-# Clone singlefile and move it to the /bin folder so archivebox can find it
+# Install Python dependencies
-
+WORKDIR "$CODE_DIR"
-WORKDIR "$EXTRA_PATH"
-RUN wget -qO - https://github.com/gildas-lormeau/SingleFile/archive/master.zip > SingleFile.zip \
-    && unzip -q SingleFile.zip \
-    && npm install --prefix SingleFile-master/cli --production > /dev/null 2>&1 \
-    && chmod +x SingleFile-master/cli/single-file 
-
-# Run everything from here on out as non-privileged user
-RUN groupadd --system archivebox \
-    && useradd --system --create-home --gid archivebox --groups audio,video archivebox
-
-ADD . "$CODE_PATH"
-WORKDIR "$CODE_PATH"
 ENV PATH="${PATH}:$VENV_PATH/bin"
 RUN python -m venv --clear --symlinks "$VENV_PATH" \
-    && pip install --upgrade pip setuptools \
+    && pip install --upgrade --quiet pip setuptools
-    && pip install -e .
+ADD ./archivebox.egg-info/requires.txt "$CODE_DIR/archivebox.egg-info/requires.txt"
+RUN grep -B 1000 -E '^$' "$CODE_DIR/archivebox.egg-info/requires.txt" | pip install --quiet -r /dev/stdin
 
-VOLUME "$DATA_PATH"
+# Install Node dependencies
-WORKDIR "$DATA_PATH"
+WORKDIR "$NODE_DIR"
-EXPOSE 8000
+ENV PATH="${PATH}:$NODE_DIR/node_modules/.bin" \
+    npm_config_loglevel=error
+RUN npm install -g npm
+ADD ./package.json ./package.json
+RUN npm install
+
+# Install ArchiveBox Python package
+WORKDIR "$CODE_DIR"
+ADD . "$CODE_DIR"
+RUN pip install -e .
+
+# Setup ArchiveBox runtime config
+WORKDIR "$DATA_DIR"
 ENV IN_DOCKER=True \
-    CHROME_BINARY=chromium \
     CHROME_SANDBOX=False \
-    SINGLEFILE_BINARY="$EXTRA_PATH/SingleFile-master/cli/single-file"
+    CHROME_BINARY="chromium" \
+    SINGLEFILE_BINARY="$NODE_DIR/node_modules/.bin/single-file" \
+    READABILITY_BINARY="$NODE_DIR/node_modules/.bin/readability-extractor"
 
-RUN env ALLOW_ROOT=True archivebox version
+# Print version for nice docker finish summary
+RUN archivebox version
+
+# Open up the interfaces to the outside world
+VOLUME "$DATA_DIR"
+VOLUME "$CODE_DIR"
+EXPOSE 8000
 
 ENTRYPOINT ["dumb-init", "--", "/app/bin/docker_entrypoint.sh"]
 CMD ["archivebox", "server", "0.0.0.0:8000"]

bin/archive

@@ -1,7 +1,11 @@
-#!/bin/sh
+#!/bin/bash
 
-echo "[X] This method of running ArchiveBox is deprecated as of >= v0.4."
+if python3 -m django --version >/dev/null 2>&1; then
-echo "    You should 'pip install archivebox' and use the installed 'archivebox' binary instead."
+    python3 -m archivebox "$*"
-echo "    For more info, see the Quickstart section of the README.md:"
+else
-echo "        https://github.com/pirate/ArchiveBox#Quickstart"
+    echo '[X] ArchiveBox must be installed before using:'
-exit 2
+    echo "    pip install archivebox"
+    echo
+    echo "Hint: Did you forget to activate a virtuenv or set your $$PATH?"
+    exit 2
+fi

package.json

@@ -0,0 +1,17 @@
+{
+	"name": "archivebox",
+	"version": "0.4.14",
+	"description": "ArchiveBox: The self-hosted internet archive",
+	"author": "Nick Sweeting <archivebox-npm@sweeting.me>",
+	"license": "MIT",
+	"scripts": {
+		"archivebox": "./bin/archive"
+	},
+	"bin": {
+		"archivebox": "./bin/archive"
+	},
+	"dependencies": {
+		"readability-extractor": "git+https://github.com/pirate/readability-extractor.git",
+		"single-file": "https://gitpkg.now.sh/pirate/SingleFile/cli?66c7b9bc"
+	}
+}