archivebox/.github/workflows/test.yml

name: 'Lint, Test, and Build'
on: [push]

env:
  MAX_LINE_LENGTH: 110
  DOCKER_IMAGE: archivebox-ci

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2

      - name: Set up Python
        uses: actions/setup-python@v1
        with:
          python-version: 3.8
          architecture: x64

      - name: Install flake8
        run: |
          pip install flake8          

      - name: Lint with flake8
        run: |
          # one pass for show-stopper syntax errors or undefined names
          flake8 archivebox --count --show-source --statistics
          # one pass for small stylistic things
          flake8 archivebox --count --max-line-length="$MAX_LINE_LENGTH" --statistics          

  test:
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: [ubuntu-latest, macos-latest]
        python: [3.7, 3.8]

    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1

      ### Setup Python & JS Languages
      - name: Set up Python ${{ matrix.python }}
        uses: actions/setup-python@v1
        with:
          python-version: ${{ matrix.python }}
          architecture: x64

      - name: Set up Node JS 14.7.0
        uses: actions/setup-node@v1
        with:
          node-version: 14.7.0

      ### Install Python & JS Dependencies
      - name: Get pip cache dir
        id: pip-cache
        run: |
          echo "::set-output name=dir::$(pip cache dir)"          

      - name: Cache pip
        uses: actions/cache@v2
        id: cache-pip
        with:
          path: ${{ steps.pip-cache.outputs.dir }}
          key: ${{ runner.os }}-${{ matrix.python }}-venv-${{ hashFiles('setup.py') }}
          restore-keys: |
            ${{ runner.os }}-${{ matrix.python }}-venv-            

      - name: Install pip dependencies
        run: |
          python -m pip install pytest bottle wheel setuptools
          python -m pip install -e .          

      - name: Get npm cache dir
        id: npm-cache
        run: |
          echo "::set-output name=dir::$GITHUB_WORKSPACE/node_modules"          

      - name: Cache npm
        uses: actions/cache@v2
        id: cache-npm
        with:
          path: ${{ steps.npm-cache.outputs.dir }}
          key: ${{ runner.os }}-node_modules-${{ hashFiles('package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-node_modules            

      - name: Install npm requirements
        run: |
          npm install
          echo "SINGLEFILE_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/single-file" >> $GITHUB_ENV
          echo "READABILITY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/readability-extractor" >> $GITHUB_ENV
          echo "MERCURY_BINARY=$GITHUB_WORKSPACE/node_modules/.bin/mercury-parser" >> $GITHUB_ENV          

      ### Run the tests
      - name: Directory listing for debugging
        run: |
          pwd
          ls -a ./
          archivebox version          

      - name: Test built package with pytest
        run: |
          python -m pytest -s          

  docker-test:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 1

      # TODO: as of 2020-11 this helper layer broke, upgrade and re-enable this once it's usable again
      # - uses: satackey/action-docker-layer-caching@v0.0.8

      - name: Build image
        run: |
          docker build . -t "$DOCKER_IMAGE"          

      - name: Init data dir
        run: |
          mkdir data
          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" init          

      - name: Run test server
        run: |
          sudo bash -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
          docker run --name www-nginx -p 80:80 -d nginx          

      - name: Add link
        run: |
          docker run -v "$PWD"/data:/data --network host "$DOCKER_IMAGE" add http://www.test-nginx-1.local          

      - name: Add stdin link
        run: |
          echo "http://www.test-nginx-2.local" | docker run -i --network host -v "$PWD"/data:/data "$DOCKER_IMAGE" add          

      - name: List links
        run: |
          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-1.local" || { echo "The site 1 isn't in the list"; exit 1; }
          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-2.local" || { echo "The site 2 isn't in the list"; exit 1; }          

      - name: Start docker-compose stack
        run: |
          docker-compose run archivebox init
          docker-compose up -d
          sleep 5
          curl --silent --location 'http://127.0.0.1:8000' | grep 'ArchiveBox'
          curl --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js' | grep 'window.django'          

      - name: Check added urls show up in index
        run: |
          docker-compose run archivebox add 'http://example.com/#test_docker' --index-only
          curl --silent --location 'http://127.0.0.1:8000' | grep 'http://example.com/#test_docker'
          docker-compose down || true