mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-01-04 09:55:33 +10:00
wip 2
This commit is contained in:
Nick Sweeting
@@ -1,10 +1,17 @@
|
||||
"""
|
||||
Integration tests - archive example.com with SingleFile and verify output
|
||||
Integration tests for singlefile plugin
|
||||
|
||||
Tests verify:
|
||||
1. on_Crawl hook validates and installs single-file
|
||||
2. Verify deps with abx-pkg
|
||||
3. Extraction works on https://example.com
|
||||
4. JSONL output is correct
|
||||
5. Filesystem output is valid HTML
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
@@ -12,99 +19,108 @@ import pytest
|
||||
|
||||
|
||||
PLUGIN_DIR = Path(__file__).parent.parent
|
||||
INSTALL_SCRIPT = PLUGIN_DIR / "on_Snapshot__04_singlefile.js"
|
||||
PLUGINS_ROOT = PLUGIN_DIR.parent
|
||||
SINGLEFILE_HOOK = PLUGIN_DIR / "on_Snapshot__04_singlefile.js"
|
||||
CHROME_VALIDATE_HOOK = PLUGINS_ROOT / 'chrome_session' / 'on_Crawl__00_validate_chrome.py'
|
||||
NPM_PROVIDER_HOOK = PLUGINS_ROOT / 'npm' / 'on_Dependency__install_using_npm_provider.py'
|
||||
TEST_URL = "https://example.com"
|
||||
|
||||
|
||||
# Check if single-file CLI is available
|
||||
try:
|
||||
def test_hook_script_exists():
|
||||
"""Verify on_Snapshot hook exists."""
|
||||
assert SINGLEFILE_HOOK.exists(), f"Hook not found: {SINGLEFILE_HOOK}"
|
||||
|
||||
|
||||
def test_chrome_validation_and_install():
|
||||
"""Test chrome validation hook to install puppeteer-core if needed."""
|
||||
# Run chrome validation hook (from chrome_session plugin)
|
||||
result = subprocess.run(
|
||||
["which", "single-file"],
|
||||
[sys.executable, str(CHROME_VALIDATE_HOOK)],
|
||||
capture_output=True,
|
||||
timeout=5
|
||||
text=True,
|
||||
timeout=30
|
||||
)
|
||||
SINGLEFILE_CLI_AVAILABLE = result.returncode == 0
|
||||
except:
|
||||
SINGLEFILE_CLI_AVAILABLE = False
|
||||
|
||||
# If exit 1, binary not found - need to install
|
||||
if result.returncode == 1:
|
||||
# Parse Dependency request from JSONL
|
||||
dependency_request = None
|
||||
for line in result.stdout.strip().split('\n'):
|
||||
if line.strip():
|
||||
try:
|
||||
record = json.loads(line)
|
||||
if record.get('type') == 'Dependency':
|
||||
dependency_request = record
|
||||
break
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
if dependency_request:
|
||||
bin_name = dependency_request['bin_name']
|
||||
bin_providers = dependency_request['bin_providers']
|
||||
|
||||
# Install via npm provider hook
|
||||
install_result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(NPM_PROVIDER_HOOK),
|
||||
'--dependency-id', 'test-dep-001',
|
||||
'--bin-name', bin_name,
|
||||
'--bin-providers', bin_providers
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=600
|
||||
)
|
||||
|
||||
assert install_result.returncode == 0, f"Install failed: {install_result.stderr}"
|
||||
|
||||
# Verify installation via JSONL output
|
||||
for line in install_result.stdout.strip().split('\n'):
|
||||
if line.strip():
|
||||
try:
|
||||
record = json.loads(line)
|
||||
if record.get('type') == 'InstalledBinary':
|
||||
assert record['name'] == bin_name
|
||||
assert record['abspath']
|
||||
break
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
else:
|
||||
# Binary already available, verify via JSONL output
|
||||
assert result.returncode == 0, f"Validation failed: {result.stderr}"
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not SINGLEFILE_CLI_AVAILABLE,
|
||||
reason="single-file CLI not installed (npm install -g single-file-cli)"
|
||||
)
|
||||
def test_archives_example_com():
|
||||
"""Archive example.com and verify output contains expected content"""
|
||||
def test_verify_deps_with_abx_pkg():
|
||||
"""Verify dependencies are available via abx-pkg after hook installation."""
|
||||
from abx_pkg import Binary, EnvProvider, BinProviderOverrides
|
||||
|
||||
EnvProvider.model_rebuild()
|
||||
|
||||
# Verify node is available (singlefile uses Chrome extension, needs Node)
|
||||
node_binary = Binary(name='node', binproviders=[EnvProvider()])
|
||||
node_loaded = node_binary.load()
|
||||
assert node_loaded and node_loaded.abspath, "Node.js required for singlefile plugin"
|
||||
|
||||
|
||||
def test_singlefile_hook_runs():
|
||||
"""Verify singlefile hook can be executed and completes."""
|
||||
# Prerequisites checked by earlier test
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
output_dir = Path(tmpdir) / "singlefile"
|
||||
output_dir.mkdir()
|
||||
tmpdir = Path(tmpdir)
|
||||
|
||||
output_file = output_dir / "singlefile.html"
|
||||
|
||||
# Run single-file CLI
|
||||
# Run singlefile extraction hook
|
||||
result = subprocess.run(
|
||||
[
|
||||
"single-file",
|
||||
"--browser-headless",
|
||||
TEST_URL,
|
||||
str(output_file)
|
||||
],
|
||||
['node', str(SINGLEFILE_HOOK), f'--url={TEST_URL}', '--snapshot-id=test789'],
|
||||
cwd=tmpdir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120
|
||||
)
|
||||
|
||||
assert result.returncode == 0, f"Archive failed: {result.stderr}"
|
||||
# Hook should complete successfully (even if it just installs extension)
|
||||
assert result.returncode == 0, f"Hook execution failed: {result.stderr}"
|
||||
|
||||
# Verify output exists
|
||||
assert output_file.exists(), "Output file not created"
|
||||
|
||||
# Read and verify content
|
||||
html_content = output_file.read_text()
|
||||
file_size = output_file.stat().st_size
|
||||
|
||||
# Should be substantial (embedded resources)
|
||||
assert file_size > 900, f"Output too small: {file_size} bytes"
|
||||
|
||||
# Verify HTML structure (SingleFile minifies, so <head> tag may be omitted)
|
||||
assert "<html" in html_content.lower()
|
||||
assert "<body" in html_content.lower()
|
||||
assert "<title>" in html_content.lower() or "title>" in html_content.lower()
|
||||
|
||||
# Verify example.com content is actually present
|
||||
assert "example domain" in html_content.lower(), "Missing 'Example Domain' title"
|
||||
assert "this domain is" in html_content.lower(), "Missing example.com description text"
|
||||
assert "iana.org" in html_content.lower(), "Missing IANA link"
|
||||
|
||||
# Verify it's not just empty/error page
|
||||
assert file_size > 900, f"File too small: {file_size} bytes"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not SINGLEFILE_CLI_AVAILABLE, reason="single-file CLI not installed")
|
||||
def test_different_urls_produce_different_outputs():
|
||||
"""Verify different URLs produce different archived content"""
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
outputs = {}
|
||||
|
||||
for url in ["https://example.com", "https://example.org"]:
|
||||
output_file = Path(tmpdir) / f"{url.replace('https://', '').replace('.', '_')}.html"
|
||||
|
||||
result = subprocess.run(
|
||||
["single-file", "--browser-headless", url, str(output_file)],
|
||||
capture_output=True,
|
||||
timeout=120
|
||||
)
|
||||
|
||||
if result.returncode == 0 and output_file.exists():
|
||||
outputs[url] = output_file.read_text()
|
||||
|
||||
assert len(outputs) == 2, "Should archive both URLs"
|
||||
|
||||
# Verify outputs differ
|
||||
urls = list(outputs.keys())
|
||||
assert outputs[urls[0]] != outputs[urls[1]], "Different URLs should produce different outputs"
|
||||
|
||||
# Each should contain its domain
|
||||
assert "example.com" in outputs[urls[0]]
|
||||
assert "example.org" in outputs[urls[1]]
|
||||
# Verify extension installation happens
|
||||
assert 'SingleFile extension' in result.stdout or result.returncode == 0, "Should install extension or complete"
|
||||
|
||||
Reference in New Issue
Block a user