diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 2749a22f..93b185eb 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -3,6 +3,14 @@ name: Build Debian package on: workflow_dispatch: workflow_call: + push: + branches: ['**'] + paths: + - 'pkg/debian/**' + - 'bin/build_deb.sh' + - 'bin/release_deb.sh' + - '.github/workflows/debian.yml' + - 'pyproject.toml' release: types: [published] @@ -27,7 +35,6 @@ jobs: - name: Install nfpm env: - # Pin nfpm version for reproducible builds NFPM_VERSION: "2.45.1" run: | curl -fsSL "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_x86_64.tar.gz" \ @@ -70,6 +77,29 @@ jobs: runs-on: ubuntu-24.04 steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + + - name: Install build dependencies + uses: awalsh128/cache-apt-pkgs-action@latest + with: + packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 + version: 1.0 + + - name: Build local wheel + run: | + uv sync --frozen --all-extras --no-install-project --no-install-workspace + uv build --wheel --out-dir /tmp/wheels/ + - name: Download .deb artifact uses: actions/download-artifact@v4 with: @@ -79,29 +109,39 @@ jobs: - name: Install system dependencies run: | sudo apt-get update -qq - sudo apt-get install -y python3 python3-pip python3-venv nodejs npm git wget curl ripgrep + sudo apt-get install -y python3.13 python3.13-venv python3-pip nodejs npm git wget curl ripgrep + + - name: Pre-seed virtualenv with local wheel before dpkg install + run: | + # Create the venv and install from local wheel BEFORE dpkg runs postinstall. + # This way the postinstall's pip install either succeeds (release) or + # finds archivebox already installed (CI) and just upgrades deps. + sudo mkdir -p /opt/archivebox + sudo python3.13 -m venv /opt/archivebox/venv + sudo /opt/archivebox/venv/bin/python3 -m pip install --quiet --upgrade pip setuptools + sudo /opt/archivebox/venv/bin/pip install --quiet /tmp/wheels/archivebox-*.whl + echo "[√] Pre-seeded /opt/archivebox/venv with local wheel" - name: Install .deb package run: | + # dpkg install will run postinstall which creates the user, + # sets up systemd, and tries pip install (which finds it already installed) sudo dpkg -i archivebox*.deb || sudo apt-get install -f -y - - name: Run postinstall setup - run: | - # The postinstall script creates the venv and pip-installs archivebox - # If it didn't run during dpkg install, run it manually - if [ ! -f /opt/archivebox/venv/bin/archivebox ]; then - sudo /opt/archivebox/install.sh - fi - - name: Verify archivebox is installed run: | which archivebox archivebox version - - name: Test archivebox init + - name: Verify wrapper script works run: | - # Create a test data directory and init - sudo useradd -r -s /bin/bash -d /var/lib/archivebox archivebox 2>/dev/null || true + /usr/bin/archivebox version + /usr/bin/archivebox --help | head -5 + + - name: Test archivebox init as archivebox user + run: | + # The postinstall should have created the user + id archivebox sudo mkdir -p /tmp/archivebox-test sudo chown archivebox:archivebox /tmp/archivebox-test sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init --setup' @@ -109,3 +149,12 @@ jobs: - name: Test archivebox status run: | sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox status' + + - name: Test archivebox add + run: | + sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox add --parser url_list "https://example.com"' || true + + - name: Verify systemd service file exists + run: | + test -f /usr/lib/systemd/system/archivebox.service + cat /usr/lib/systemd/system/archivebox.service diff --git a/.github/workflows/homebrew.yml b/.github/workflows/homebrew.yml index 50877a20..19098b2b 100644 --- a/.github/workflows/homebrew.yml +++ b/.github/workflows/homebrew.yml @@ -3,6 +3,14 @@ name: Build Homebrew formula on: workflow_dispatch: workflow_call: + push: + branches: ['**'] + paths: + - 'brew_dist/**' + - 'bin/build_brew.sh' + - 'bin/release_brew.sh' + - '.github/workflows/homebrew.yml' + - 'pyproject.toml' release: types: [published] @@ -10,7 +18,161 @@ permissions: contents: read jobs: - build: + build-and-test: + strategy: + fail-fast: false + matrix: + os: [macos-latest, ubuntu-24.04] + runs-on: ${{ matrix.os }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Get version + id: version + run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT" + + - name: Validate formula template syntax + run: ruby -c brew_dist/archivebox.rb + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + + - name: Install build dependencies (Linux) + if: runner.os == 'Linux' + uses: awalsh128/cache-apt-pkgs-action@latest + with: + packages: build-essential python3-dev python3-setuptools libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 + version: 1.0 + + - name: Build local sdist + run: | + uv sync --frozen --all-extras --no-install-project --no-install-workspace + uv build --sdist --out-dir /tmp/sdist/ + + - name: Generate formula from local sdist + run: | + VERSION="${{ steps.version.outputs.version }}" + SDIST_PATH="$(ls /tmp/sdist/archivebox-*.tar.gz | head -1)" + SDIST_SHA256="$(shasum -a 256 "$SDIST_PATH" | awk '{print $1}')" + + # Install archivebox + poet into a temp venv to generate resource stanzas + python3 -m venv /tmp/poet-venv + source /tmp/poet-venv/bin/activate + pip install --quiet "$SDIST_PATH" homebrew-pypi-poet + echo "[+] Generating resource stanzas with homebrew-pypi-poet..." + RESOURCES="$(poet archivebox)" + deactivate + + # For CI: use file:// URL pointing to local sdist + # For release: this gets overridden with the PyPI URL + SDIST_URL="file://${SDIST_PATH}" + + cat > /tmp/archivebox.rb << RUBY +class Archivebox < Formula + include Language::Python::Virtualenv + + desc "Self-hosted internet archiving solution" + homepage "https://github.com/ArchiveBox/ArchiveBox" + url "${SDIST_URL}" + sha256 "${SDIST_SHA256}" + license "MIT" + + depends_on "python@3.13" + depends_on "node" + depends_on "git" + depends_on "wget" + depends_on "curl" + depends_on "ripgrep" + depends_on "yt-dlp" + + on_linux do + depends_on "pkg-config" => :build + depends_on "openssl@3" + depends_on "libffi" + end + +${RESOURCES} + + def install + virtualenv_install_with_resources + end + + def post_install + system bin/"archivebox", "install", "--binproviders", "pip,npm" + end + + service do + run [opt_bin/"archivebox", "server", "--quick-init", "0.0.0.0:8000"] + keep_alive crashed: true + working_dir var/"archivebox" + log_path var/"log/archivebox.log" + error_log_path var/"log/archivebox.log" + end + + test do + assert_match version.to_s, shell_output("#{bin}/archivebox version") + end +end +RUBY + + echo "[√] Generated formula:" + ruby -c /tmp/archivebox.rb + cat /tmp/archivebox.rb + + - name: Install Homebrew (Linux only) + if: runner.os == 'Linux' + run: | + NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo '/home/linuxbrew/.linuxbrew/bin' >> "$GITHUB_PATH" + + - name: Set up brew shellenv (Linux only) + if: runner.os == 'Linux' + run: | + eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" + echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >> "$GITHUB_ENV" + echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >> "$GITHUB_ENV" + echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >> "$GITHUB_ENV" + echo "$HOMEBREW_PREFIX/bin" >> "$GITHUB_PATH" + echo "$HOMEBREW_PREFIX/sbin" >> "$GITHUB_PATH" + + - name: Install brew dependencies + run: | + brew install python@3.13 node git wget curl ripgrep yt-dlp + if [ "$RUNNER_OS" = "Linux" ]; then + brew install pkg-config openssl@3 libffi + fi + + - name: Install archivebox via brew from local formula + run: | + brew install --build-from-source --verbose /tmp/archivebox.rb + + - name: Verify archivebox version + run: archivebox version + + - name: Test archivebox init + run: | + mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test + archivebox init --setup + + - name: Test archivebox status + run: | + cd /tmp/archivebox-test + archivebox status + + # On release only: generate the real formula with PyPI URL and push to tap + release: + if: github.event_name == 'release' + needs: build-and-test runs-on: macos-latest steps: @@ -28,7 +190,6 @@ jobs: python-version: "3.13" - name: Wait for PyPI package availability - if: github.event_name == 'release' run: | VERSION="${{ steps.version.outputs.version }}" echo "[+] Waiting for archivebox==${VERSION} to be available on PyPI..." @@ -45,7 +206,7 @@ jobs: sleep 30 done - - name: Generate Homebrew formula + - name: Generate release formula with PyPI URL run: | VERSION="${{ steps.version.outputs.version }}" python3 -m venv /tmp/poet-venv @@ -74,61 +235,64 @@ jobs: deactivate - # Generate the formula file cat > /tmp/archivebox.rb << RUBY - # Auto-generated Homebrew formula for archivebox ${VERSION} - # Generated by GitHub Actions homebrew workflow using homebrew-pypi-poet +# Auto-generated Homebrew formula for archivebox ${VERSION} +# Generated by GitHub Actions on release using homebrew-pypi-poet +# +# Users install with: +# brew tap archivebox/archivebox +# brew install archivebox - class Archivebox < Formula - include Language::Python::Virtualenv +class Archivebox < Formula + include Language::Python::Virtualenv - desc "Self-hosted internet archiving solution" - homepage "https://github.com/ArchiveBox/ArchiveBox" - url "${SDIST_URL}" - sha256 "${SDIST_SHA256}" - license "MIT" - head "https://github.com/ArchiveBox/ArchiveBox.git", branch: "dev" + desc "Self-hosted internet archiving solution" + homepage "https://github.com/ArchiveBox/ArchiveBox" + url "${SDIST_URL}" + sha256 "${SDIST_SHA256}" + license "MIT" + head "https://github.com/ArchiveBox/ArchiveBox.git", branch: "dev" - depends_on "python@3.13" - depends_on "node" - depends_on "git" - depends_on "wget" - depends_on "curl" - depends_on "ripgrep" - depends_on "yt-dlp" + depends_on "python@3.13" + depends_on "node" + depends_on "git" + depends_on "wget" + depends_on "curl" + depends_on "ripgrep" + depends_on "yt-dlp" - on_linux do - depends_on "pkg-config" => :build - depends_on "openssl@3" - depends_on "libffi" - end + on_linux do + depends_on "pkg-config" => :build + depends_on "openssl@3" + depends_on "libffi" + end - # Python dependency resource blocks auto-generated by homebrew-pypi-poet - ${RESOURCES} +${RESOURCES} - def install - virtualenv_install_with_resources - end + def install + virtualenv_install_with_resources + end - def post_install - system bin/"archivebox", "install", "--binproviders", "pip,npm" - end + def post_install + system bin/"archivebox", "install", "--binproviders", "pip,npm" + end - service do - run [opt_bin/"archivebox", "server", "--quick-init", "0.0.0.0:8000"] - keep_alive crashed: true - working_dir var/"archivebox" - log_path var/"log/archivebox.log" - error_log_path var/"log/archivebox.log" - end + service do + run [opt_bin/"archivebox", "server", "--quick-init", "0.0.0.0:8000"] + keep_alive crashed: true + working_dir var/"archivebox" + log_path var/"log/archivebox.log" + error_log_path var/"log/archivebox.log" + end - test do - assert_match version.to_s, shell_output("#{bin}/archivebox version") - end - end - RUBY + test do + assert_match version.to_s, shell_output("#{bin}/archivebox version") + end +end +RUBY - echo "[√] Generated formula at /tmp/archivebox.rb" + echo "[√] Generated release formula:" + ruby -c /tmp/archivebox.rb cat /tmp/archivebox.rb - name: Upload formula artifact @@ -137,15 +301,18 @@ jobs: name: archivebox.rb path: /tmp/archivebox.rb + - name: Test formula install + run: | + brew install --build-from-source /tmp/archivebox.rb + archivebox version + - name: Push to homebrew-archivebox tap - if: github.event_name == 'release' env: GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }} run: | VERSION="${{ steps.version.outputs.version }}" git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap - # Use the generated formula (with real resources), NOT the template cp /tmp/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \ cp /tmp/archivebox.rb /tmp/tap/archivebox.rb @@ -160,76 +327,3 @@ jobs: git push origin HEAD echo "[√] Formula pushed to homebrew-archivebox tap" fi - - test-macos: - needs: build - runs-on: macos-latest - - steps: - - uses: actions/checkout@v4 - - - name: Download formula artifact - uses: actions/download-artifact@v4 - with: - name: archivebox.rb - path: /tmp/ - - - name: Install dependencies - run: | - brew install python@3.13 node git wget curl ripgrep yt-dlp - - - name: Test Homebrew formula install - run: | - brew install --build-from-source /tmp/archivebox.rb - - - name: Verify archivebox CLI - run: | - archivebox version - - - name: Test archivebox init - run: | - mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test - archivebox init --setup - archivebox status - - test-linux: - needs: build - runs-on: ubuntu-24.04 - - steps: - - uses: actions/checkout@v4 - - - name: Download formula artifact - uses: actions/download-artifact@v4 - with: - name: archivebox.rb - path: /tmp/ - - - name: Install Homebrew on Linux - run: | - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" - echo >> "$GITHUB_ENV" - echo 'HOMEBREW_PREFIX=/home/linuxbrew/.linuxbrew' >> "$GITHUB_ENV" - echo '/home/linuxbrew/.linuxbrew/bin' >> "$GITHUB_PATH" - - - name: Install dependencies - run: | - eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" - brew install python@3.13 node git wget curl ripgrep yt-dlp pkg-config openssl@3 libffi - - - name: Test Homebrew formula install - run: | - eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" - brew install --build-from-source /tmp/archivebox.rb - - - name: Verify archivebox CLI - run: | - eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" - archivebox version - - - name: Test archivebox init - run: | - eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" - mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test - archivebox init --setup - archivebox status diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 41f1638c..86a507fa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,6 +4,9 @@ name: Release # 1. Build and publish pip package to PyPI # 2. Build .deb packages and Homebrew formula (in parallel, after pip) # 3. Build Docker images (in parallel with deb/brew) +# +# Individual workflows also run on push for CI (see their own triggers). +# This workflow ensures the correct ordering during a release. on: workflow_dispatch: