Wire up GitHub Actions for deb/brew build, test, and release

- Fix debian.yml: pin nfpm version, add permissions, improve test job
  with user creation, init test, and status check
- Fix homebrew.yml: use PyPI JSON API (macOS-compatible, no grep -oP),
  wait for PyPI availability on release, use generated formula not template,
  add Linux (Linuxbrew) test job alongside macOS
- Add release.yml orchestrator: pip → deb + brew + docker in order
- Add workflow_call triggers to pip.yml and docker.yml
- Fix build_brew.sh: replace grep -oP with Python-based PyPI API,
  add on_linux deps (pkg-config, openssl, libffi)
- Fix setup.sh: use GitHub API to find correct .deb download URL
  (filename includes version number)
- Fix postinstall.sh: create archivebox system user, pin version from
  package, check for systemd before daemon-reload
- Fix preremove.sh: stop service before removal, check for systemd
- Fix install.sh: fallback to latest if pinned version not on PyPI
- Add on_linux deps to brew formula for Linuxbrew support
- Tested: .deb builds, installs, creates user, runs archivebox init

https://claude.ai/code/session_01Vx1EsNrNySgsc8Y67dGzCn

.github/workflows/debian.yml

@@ -2,9 +2,13 @@ name: Build Debian package
 
 on:
   workflow_dispatch:
+  workflow_call:
   release:
     types: [published]
 
+permissions:
+  contents: write
+
 jobs:
   build:
     runs-on: ubuntu-24.04
@@ -22,8 +26,13 @@ jobs:
         run: echo "version=$(grep '^version = ' pyproject.toml | awk -F'\"' '{print $2}')" >> "$GITHUB_OUTPUT"
 
       - name: Install nfpm
+        env:
+          # Pin nfpm version for reproducible builds
+          NFPM_VERSION: "2.45.1"
         run: |
-          curl -sfL https://install.goreleaser.com/github.com/goreleaser/nfpm.sh | sh -s -- -b /usr/local/bin
+          curl -fsSL "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${NFPM_VERSION}_Linux_x86_64.tar.gz" \
+            | sudo tar -xz -C /usr/local/bin nfpm
+          nfpm --version
 
       - name: Build .deb package
         run: |
@@ -31,6 +40,18 @@ jobs:
           export ARCH="${{ matrix.arch }}"
           ./bin/build_deb.sh
 
+      - name: Verify .deb package contents
+        run: |
+          DEB_FILE="$(ls dist/archivebox*.deb | head -1)"
+          echo "=== Package info ==="
+          dpkg-deb --info "$DEB_FILE"
+          echo ""
+          echo "=== Package contents ==="
+          dpkg-deb --contents "$DEB_FILE"
+          echo ""
+          echo "=== Control fields ==="
+          dpkg-deb --field "$DEB_FILE"
+
       - name: Upload .deb artifact
         uses: actions/upload-artifact@v4
         with:
@@ -55,14 +76,36 @@ jobs:
           pattern: archivebox-*-amd64.deb
           merge-multiple: true
 
-      - name: Install .deb package
+      - name: Install system dependencies
         run: |
           sudo apt-get update -qq
           sudo apt-get install -y python3 python3-pip python3-venv nodejs npm git wget curl ripgrep
+
+      - name: Install .deb package
+        run: |
           sudo dpkg -i archivebox*.deb || sudo apt-get install -f -y
 
-      - name: Test archivebox CLI
+      - name: Run postinstall setup
         run: |
+          # The postinstall script creates the venv and pip-installs archivebox
+          # If it didn't run during dpkg install, run it manually
+          if [ ! -f /opt/archivebox/venv/bin/archivebox ]; then
+            sudo /opt/archivebox/install.sh
+          fi
+
+      - name: Verify archivebox is installed
+        run: |
+          which archivebox
           archivebox version
-          mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test
-          archivebox init --setup
+
+      - name: Test archivebox init
+        run: |
+          # Create a test data directory and init
+          sudo useradd -r -s /bin/bash -d /var/lib/archivebox archivebox 2>/dev/null || true
+          sudo mkdir -p /tmp/archivebox-test
+          sudo chown archivebox:archivebox /tmp/archivebox-test
+          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox init --setup'
+
+      - name: Test archivebox status
+        run: |
+          sudo -u archivebox bash -c 'cd /tmp/archivebox-test && /opt/archivebox/venv/bin/archivebox status'

.github/workflows/docker.yml

@@ -2,6 +2,7 @@ name: Build Docker image
 
 on:
   workflow_dispatch:
+  workflow_call:
   push:
     branches:
       - '**'

.github/workflows/homebrew.yml

@@ -2,9 +2,13 @@ name: Build Homebrew formula
 
 on:
   workflow_dispatch:
+  workflow_call:
   release:
     types: [published]
 
+permissions:
+  contents: read
+
 jobs:
   build:
     runs-on: macos-latest
@@ -23,36 +27,141 @@ jobs:
         with:
           python-version: "3.13"
 
+      - name: Wait for PyPI package availability
+        if: github.event_name == 'release'
+        run: |
+          VERSION="${{ steps.version.outputs.version }}"
+          echo "[+] Waiting for archivebox==${VERSION} to be available on PyPI..."
+          for i in $(seq 1 30); do
+            if pip index versions archivebox 2>/dev/null | grep -q "$VERSION"; then
+              echo "[√] archivebox==${VERSION} is available on PyPI"
+              break
+            fi
+            if [ "$i" -eq 30 ]; then
+              echo "[!] Timed out waiting for PyPI. Trying to install anyway..."
+              break
+            fi
+            echo "    Attempt $i/30 - not yet available, waiting 30s..."
+            sleep 30
+          done
+
       - name: Generate Homebrew formula
         run: |
+          VERSION="${{ steps.version.outputs.version }}"
           python3 -m venv /tmp/poet-venv
           source /tmp/poet-venv/bin/activate
-          pip install --quiet "archivebox==${{ steps.version.outputs.version }}" homebrew-pypi-poet
-          poet -f archivebox > /tmp/archivebox-generated.rb
+
+          pip install --quiet "archivebox==${VERSION}" homebrew-pypi-poet
+
+          echo "[+] Generating resource stanzas with homebrew-pypi-poet..."
+          RESOURCES="$(poet archivebox)"
+
+          # Get sdist URL and SHA256 from PyPI JSON API
+          PYPI_JSON="$(curl -fsSL "https://pypi.org/pypi/archivebox/${VERSION}/json" 2>/dev/null || echo '')"
+          SDIST_URL=""
+          SDIST_SHA256=""
+          if [ -n "$PYPI_JSON" ]; then
+            SDIST_URL="$(echo "$PYPI_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(next((u['url'] for u in d['urls'] if u['packagetype']=='sdist'), ''))" 2>/dev/null || echo '')"
+            SDIST_SHA256="$(echo "$PYPI_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(next((u['digests']['sha256'] for u in d['urls'] if u['packagetype']=='sdist'), ''))" 2>/dev/null || echo '')"
+          fi
+          if [ -z "$SDIST_URL" ]; then
+            SDIST_URL="https://files.pythonhosted.org/packages/source/a/archivebox/archivebox-${VERSION}.tar.gz"
+          fi
+          if [ -z "$SDIST_SHA256" ]; then
+            pip download --no-binary :all: --no-deps -d /tmp/sdist "archivebox==${VERSION}" 2>/dev/null || true
+            SDIST_SHA256="$(shasum -a 256 /tmp/sdist/*.tar.gz 2>/dev/null | awk '{print $1}' || echo '')"
+          fi
+
           deactivate
 
+          # Generate the formula file
+          cat > /tmp/archivebox.rb << RUBY
+          # Auto-generated Homebrew formula for archivebox ${VERSION}
+          # Generated by GitHub Actions homebrew workflow using homebrew-pypi-poet
+
+          class Archivebox < Formula
+            include Language::Python::Virtualenv
+
+            desc "Self-hosted internet archiving solution"
+            homepage "https://github.com/ArchiveBox/ArchiveBox"
+            url "${SDIST_URL}"
+            sha256 "${SDIST_SHA256}"
+            license "MIT"
+            head "https://github.com/ArchiveBox/ArchiveBox.git", branch: "dev"
+
+            depends_on "python@3.13"
+            depends_on "node"
+            depends_on "git"
+            depends_on "wget"
+            depends_on "curl"
+            depends_on "ripgrep"
+            depends_on "yt-dlp"
+
+            on_linux do
+              depends_on "pkg-config" => :build
+              depends_on "openssl@3"
+              depends_on "libffi"
+            end
+
+            # Python dependency resource blocks auto-generated by homebrew-pypi-poet
+          ${RESOURCES}
+
+            def install
+              virtualenv_install_with_resources
+            end
+
+            def post_install
+              system bin/"archivebox", "install", "--binproviders", "pip,npm"
+            end
+
+            service do
+              run [opt_bin/"archivebox", "server", "--quick-init", "0.0.0.0:8000"]
+              keep_alive crashed: true
+              working_dir var/"archivebox"
+              log_path var/"log/archivebox.log"
+              error_log_path var/"log/archivebox.log"
+            end
+
+            test do
+              assert_match version.to_s, shell_output("#{bin}/archivebox version")
+            end
+          end
+          RUBY
+
+          echo "[√] Generated formula at /tmp/archivebox.rb"
+          cat /tmp/archivebox.rb
+
       - name: Upload formula artifact
         uses: actions/upload-artifact@v4
         with:
           name: archivebox.rb
-          path: /tmp/archivebox-generated.rb
+          path: /tmp/archivebox.rb
 
       - name: Push to homebrew-archivebox tap
         if: github.event_name == 'release'
         env:
           GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }}
         run: |
-          # Clone the tap repo and update the formula
+          VERSION="${{ steps.version.outputs.version }}"
           git clone "https://x-access-token:${GH_TOKEN}@github.com/ArchiveBox/homebrew-archivebox.git" /tmp/tap
-          cp brew_dist/archivebox.rb /tmp/tap/archivebox.rb 2>/dev/null || cp /tmp/archivebox-generated.rb /tmp/tap/archivebox.rb
+
+          # Use the generated formula (with real resources), NOT the template
+          cp /tmp/archivebox.rb /tmp/tap/Formula/archivebox.rb 2>/dev/null || \
+            cp /tmp/archivebox.rb /tmp/tap/archivebox.rb
+
           cd /tmp/tap
           git config user.name "github-actions[bot]"
           git config user.email "github-actions[bot]@users.noreply.github.com"
-          git add archivebox.rb
-          git diff --cached --quiet || git commit -m "Update archivebox to v${{ steps.version.outputs.version }}"
-          git push origin HEAD
+          git add -A
+          if git diff --cached --quiet; then
+            echo "[i] No changes to formula, skipping push."
+          else
+            git commit -m "Update archivebox to v${VERSION}"
+            git push origin HEAD
+            echo "[√] Formula pushed to homebrew-archivebox tap"
+          fi
 
-  test:
+  test-macos:
     needs: build
     runs-on: macos-latest
 
@@ -65,7 +174,62 @@ jobs:
           name: archivebox.rb
           path: /tmp/
 
-      - name: Test Homebrew formula
+      - name: Install dependencies
+        run: |
+          brew install python@3.13 node git wget curl ripgrep yt-dlp
+
+      - name: Test Homebrew formula install
+        run: |
+          brew install --build-from-source /tmp/archivebox.rb
+
+      - name: Verify archivebox CLI
         run: |
-          brew install --build-from-source /tmp/archivebox-generated.rb || true
           archivebox version
+
+      - name: Test archivebox init
+        run: |
+          mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test
+          archivebox init --setup
+          archivebox status
+
+  test-linux:
+    needs: build
+    runs-on: ubuntu-24.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download formula artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: archivebox.rb
+          path: /tmp/
+
+      - name: Install Homebrew on Linux
+        run: |
+          /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+          echo >> "$GITHUB_ENV"
+          echo 'HOMEBREW_PREFIX=/home/linuxbrew/.linuxbrew' >> "$GITHUB_ENV"
+          echo '/home/linuxbrew/.linuxbrew/bin' >> "$GITHUB_PATH"
+
+      - name: Install dependencies
+        run: |
+          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
+          brew install python@3.13 node git wget curl ripgrep yt-dlp pkg-config openssl@3 libffi
+
+      - name: Test Homebrew formula install
+        run: |
+          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
+          brew install --build-from-source /tmp/archivebox.rb
+
+      - name: Verify archivebox CLI
+        run: |
+          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
+          archivebox version
+
+      - name: Test archivebox init
+        run: |
+          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
+          mkdir -p /tmp/archivebox-test && cd /tmp/archivebox-test
+          archivebox init --setup
+          archivebox status

.github/workflows/pip.yml

@@ -2,6 +2,7 @@ name: Build Pip package
 
 on:
   workflow_dispatch:
+  workflow_call:
   push:
     branches:
       - '**'

.github/workflows/release.yml

@@ -0,0 +1,40 @@
+name: Release
+
+# Orchestrates the full release pipeline:
+#   1. Build and publish pip package to PyPI
+#   2. Build .deb packages and Homebrew formula (in parallel, after pip)
+#   3. Build Docker images (in parallel with deb/brew)
+
+on:
+  workflow_dispatch:
+  release:
+    types: [published]
+
+permissions:
+  contents: write
+  packages: write
+  id-token: write
+
+jobs:
+  pip:
+    name: Publish to PyPI
+    uses: ./.github/workflows/pip.yml
+    secrets: inherit
+
+  debian:
+    name: Build .deb packages
+    needs: pip
+    uses: ./.github/workflows/debian.yml
+    secrets: inherit
+
+  homebrew:
+    name: Update Homebrew formula
+    needs: pip
+    uses: ./.github/workflows/homebrew.yml
+    secrets: inherit
+
+  docker:
+    name: Build Docker images
+    needs: pip
+    uses: ./.github/workflows/docker.yml
+    secrets: inherit

bin/build_brew.sh

@@ -30,12 +30,21 @@ pip install --quiet "archivebox==${VERSION}" homebrew-pypi-poet 2>/dev/null
 echo "[+] Generating resource stanzas with homebrew-pypi-poet..."
 RESOURCES="$(poet archivebox)"
 
-# Get the sdist URL and SHA256 from PyPI
-SDIST_URL="$(pip download --no-binary :all: --no-deps -d "$TMPDIR/sdist" "archivebox==${VERSION}" 2>&1 | grep -oP 'https://\S+\.tar\.gz' | head -1 || true)"
+# Get the sdist URL and SHA256 from PyPI JSON API (works on macOS and Linux)
+PYPI_JSON="$(curl -fsSL "https://pypi.org/pypi/archivebox/${VERSION}/json" 2>/dev/null || echo '')"
+if [ -n "$PYPI_JSON" ]; then
+    SDIST_URL="$(echo "$PYPI_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(next((u['url'] for u in d['urls'] if u['packagetype']=='sdist'), ''))" 2>/dev/null || echo '')"
+    SDIST_SHA256="$(echo "$PYPI_JSON" | python3 -c "import sys,json; d=json.load(sys.stdin); print(next((u['digests']['sha256'] for u in d['urls'] if u['packagetype']=='sdist'), ''))" 2>/dev/null || echo '')"
+fi
 if [ -z "$SDIST_URL" ]; then
     SDIST_URL="https://files.pythonhosted.org/packages/source/a/archivebox/archivebox-${VERSION}.tar.gz"
 fi
-SDIST_SHA256="$(pip hash "$TMPDIR/sdist/"*.tar.gz 2>/dev/null | grep 'sha256:' | cut -d: -f2 || echo '')"
+if [ -z "$SDIST_SHA256" ]; then
+    # Fallback: download and compute locally
+    mkdir -p "$TMPDIR/sdist"
+    pip download --no-binary :all: --no-deps -d "$TMPDIR/sdist" "archivebox==${VERSION}" 2>/dev/null || true
+    SDIST_SHA256="$(shasum -a 256 "$TMPDIR/sdist/"*.tar.gz 2>/dev/null | awk '{print $1}' || echo '')"
+fi
 
 deactivate
 
@@ -68,6 +77,12 @@ class Archivebox < Formula
   depends_on "ripgrep"
   depends_on "yt-dlp"
 
+  on_linux do
+    depends_on "pkg-config" => :build
+    depends_on "openssl@3"
+    depends_on "libffi"
+  end
+
   # Python dependency resource blocks auto-generated by homebrew-pypi-poet
   # AUTOGENERATED_RESOURCES_START
 ${RESOURCES}

bin/setup.sh

@@ -129,11 +129,16 @@ if which apt-get > /dev/null; then
     echo
     echo "[+] Downloading and installing ArchiveBox .deb package..."
     ARCH="$(dpkg --print-architecture)"
-    DEB_URL="https://github.com/ArchiveBox/ArchiveBox/releases/latest/download/archivebox_${ARCH}.deb"
-    curl -fsSL "$DEB_URL" -o /tmp/archivebox.deb && sudo apt install -y /tmp/archivebox.deb && rm /tmp/archivebox.deb || {
-        echo "[!] .deb install failed, falling back to pip install..."
-        sudo python3 -m pip install --upgrade archivebox yt-dlp
-    }
+    # Get the latest release .deb URL from GitHub API (filename includes version)
+    DEB_URL="$(curl -fsSL https://api.github.com/repos/ArchiveBox/ArchiveBox/releases/latest \
+        | grep -o "\"browser_download_url\": \"[^\"]*_${ARCH}\.deb\"" \
+        | head -1 | cut -d'"' -f4)" || true
+    if [ -n "$DEB_URL" ]; then
+        curl -fsSL "$DEB_URL" -o /tmp/archivebox.deb && sudo apt install -y /tmp/archivebox.deb && rm -f /tmp/archivebox.deb
+    else
+        echo "[!] Could not find .deb download URL, falling back to pip install..."
+        pip install --upgrade archivebox yt-dlp
+    fi
 # On Mac:
 elif which brew > /dev/null; then
     echo "[+] Installing ArchiveBox using Homebrew..."

brew_dist/archivebox.rb

@@ -23,6 +23,12 @@ class Archivebox < Formula
   depends_on "ripgrep"
   depends_on "yt-dlp"
 
+  on_linux do
+    depends_on "pkg-config" => :build
+    depends_on "openssl@3"
+    depends_on "libffi"
+  end
+
   # Python dependency resource blocks are auto-generated by bin/build_brew.sh
   # using homebrew-pypi-poet. Run that script to populate this section.
   # AUTOGENERATED_RESOURCES_START

pkg/debian/install.sh

@@ -20,7 +20,10 @@ fi
 # Install or upgrade archivebox
 if [ -n "$ARCHIVEBOX_VERSION" ]; then
     echo "[+] Installing archivebox==$ARCHIVEBOX_VERSION..."
-    "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade "archivebox==$ARCHIVEBOX_VERSION"
+    "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade "archivebox==$ARCHIVEBOX_VERSION" || {
+        echo "[!] archivebox==$ARCHIVEBOX_VERSION not found on PyPI, installing latest..."
+        "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade archivebox
+    }
 else
     echo "[+] Installing latest archivebox..."
     "$ARCHIVEBOX_VENV/bin/pip" install --quiet --upgrade archivebox

pkg/debian/nfpm.yaml

@@ -19,6 +19,8 @@ section: "web"
 priority: "optional"
 
 depends:
+  # python3 >= 3.11 allows .deb to install on more systems;
+  # pip enforces the actual Python version requirement from pyproject.toml
   - python3 (>= 3.11)
   - python3-pip
   - python3-venv

pkg/debian/scripts/postinstall.sh

@@ -2,4 +2,24 @@
 # postinstall script for archivebox .deb package
 set -e
 
+# Create archivebox system user if it doesn't exist
+if ! id -u archivebox >/dev/null 2>&1; then
+    useradd --system --shell /bin/bash --home-dir /var/lib/archivebox --create-home archivebox
+    echo "[+] Created archivebox system user"
+fi
+
+# Ensure data directory exists and is owned by archivebox
+mkdir -p /var/lib/archivebox
+chown archivebox:archivebox /var/lib/archivebox
+
+# Run the virtualenv install script, pinning to the .deb package version
+ARCHIVEBOX_VERSION="$(dpkg-query -W -f='${Version}' archivebox 2>/dev/null || echo '')"
+export ARCHIVEBOX_VERSION
 /opt/archivebox/install.sh
+
+# Reload systemd to pick up the service file (skip if systemd is not running)
+if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then
+    systemctl daemon-reload
+    echo "[i] To start ArchiveBox: sudo systemctl start archivebox"
+    echo "[i] To enable on boot:   sudo systemctl enable archivebox"
+fi

pkg/debian/scripts/preremove.sh

@@ -2,8 +2,17 @@
 # preremove script for archivebox .deb package
 set -e
 
+# Stop the service if running
+if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then
+    systemctl stop archivebox 2>/dev/null || true
+    systemctl disable archivebox 2>/dev/null || true
+fi
+
 echo "[+] Removing ArchiveBox virtualenv..."
 rm -rf /opt/archivebox/venv
 
-echo "[i] Your ArchiveBox data directories have NOT been removed."
-echo "    Remove them manually if you no longer need them."
+echo "[i] Your ArchiveBox data in /var/lib/archivebox has NOT been removed."
+echo "    The 'archivebox' system user has NOT been removed."
+echo "    Remove them manually if you no longer need them:"
+echo "      sudo rm -rf /var/lib/archivebox"
+echo "      sudo userdel archivebox"