ArchiveBox/.github/workflows/test.yml

name: Run tests
on: [push]

env:
  DOCKER_IMAGE: archivebox-ci
  PYTHONIOENCODING: utf-8
  PYTHONLEGACYWINDOWSSTDIO: utf-8
  USE_COLOR: False
  UV_NO_SOURCES: "1"

jobs:
  python_tests:
    runs-on: ${{ matrix.os }}
    env:
      PYTHONPATH: ${{ github.workspace }}/abx-pkg:${{ github.workspace }}/abx-plugins:${{ github.workspace }}/abx-dl

    strategy:
      matrix:
        os: [ubuntu-22.04]
        # os: [ubuntu-22.04, macos-latest, windows-latest]
        python: ["3.13"]

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
          fetch-depth: 1

      - name: Clone abx-pkg
        run: git clone --depth=1 https://github.com/ArchiveBox/abx-pkg.git abx-pkg

      - name: Clone abx-plugins
        run: git clone --depth=1 https://github.com/ArchiveBox/abx-plugins.git abx-plugins

      - name: Clone abx-dl
        run: git clone --depth=1 https://github.com/ArchiveBox/abx-dl.git abx-dl

      ### Setup Python & JS Languages
      - name: Set up Python ${{ matrix.python }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python }}
          architecture: x64

      - name: Install uv
        uses: astral-sh/setup-uv@v4
        with:
          version: "latest"

      - name: Set up Node JS
        uses: actions/setup-node@v4
        with:
          node-version: 22

      ### Install Python & JS Dependencies
      - name: Cache uv
        uses: actions/cache@v3
        with:
          path: ~/.cache/uv
          key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml', 'uv.lock') }}
          restore-keys: |
            ${{ runner.os }}-${{ matrix.python }}-uv-

      - uses: awalsh128/cache-apt-pkgs-action@latest
        with:
          packages: ripgrep build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 python3-minimal gnupg2 curl wget python3-ldap python3-msgpack python3-mutagen python3-regex python3-pycryptodome procps
          version: 1.0

      - name: Install dependencies with uv
        run: |
          uv sync --dev --all-extras --no-sources
          uv pip install -e ./abx-pkg -e ./abx-plugins[dev] -e ./abx-dl

      ### Run the tests
      - name: Directory listing for debugging
        run: |
          pwd
          ls

      - name: Archivebox version
        run: |
          mkdir -p tests/out/data
          DATA_DIR="$PWD/tests/out/data" uv run --no-sync --no-sources archivebox version

      - name: Test built package with pytest
        # TODO: remove this exception for windows once we get tests passing on that platform
        if: ${{ !contains(matrix.os, 'windows') }}
        run: |
          mkdir -p tests/out
          uv run --no-sync --no-sources pytest -s archivebox/tests --basetemp=tests/out --ignore=archivebox/pkgs

      - name: Run plugin tests
        if: ${{ !contains(matrix.os, 'windows') }}
        env:
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }}
          API_KEY_2CAPTCHA: ${{ secrets.TWOCAPTCHA_API_KEY }}
        run: |
          uv run --no-sync --no-sources bash ./bin/test_plugins.sh --no-coverage

  docker_tests:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
          fetch-depth: 1

      # TODO: as of 2020-11 this helper layer broke, upgrade and re-enable this once it's usable again
      # - uses: satackey/action-docker-layer-caching@v0.0.8

      - name: Build image
        run: |
          docker build . -t "$DOCKER_IMAGE"

      - name: Init data dir
        run: |
          mkdir "${{ github.workspace }}/data"
          docker run -v "${{ github.workspace }}/data":/data "$DOCKER_IMAGE" init

      - name: Run test server
        run: |
          sudo bash -c 'echo "127.0.0.1  www.test-nginx-1.local www.test-nginx-2.local" >> /etc/hosts'
          docker run --name www-nginx -p 80:80 -d nginx

      - name: Add link
        run: |
          docker run -v "$PWD"/data:/data --network host "$DOCKER_IMAGE" add http://www.test-nginx-1.local

      - name: Add stdin link
        run: |
          echo "http://www.test-nginx-2.local" | docker run -i --network host -v "$PWD"/data:/data "$DOCKER_IMAGE" add

      - name: List links
        run: |
          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-1.local" || { echo "The site 1 isn't in the list"; exit 1; }
          docker run -v "$PWD"/data:/data "$DOCKER_IMAGE" list | grep -q "www.test-nginx-2.local" || { echo "The site 2 isn't in the list"; exit 1; }

      - name: Start docker-compose stack
        run: |
          docker-compose run archivebox init
          docker-compose up -d
          sleep 5
          curl --silent --location 'http://127.0.0.1:8000' | grep 'ArchiveBox'
          curl --silent --location 'http://127.0.0.1:8000/static/admin/js/jquery.init.js' | grep 'window.django'

      - name: Check added urls show up in index
        run: |
          docker-compose run archivebox add 'http://example.com/#test_docker' --index-only
          curl --silent --location 'http://127.0.0.1:8000' | grep 'http://example.com/#test_docker'
          docker-compose down || true