mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-04-05 15:27:53 +10:00
way better plugin hooks system wip
This commit is contained in:
Nick Sweeting
1
archivebox/plugins/singlefile/binaries.jsonl
Normal file
1
archivebox/plugins/singlefile/binaries.jsonl
Normal file
@@ -0,0 +1 @@
|
||||
{"type": "Binary", "name": "single-file", "binproviders": "npm,env", "overrides": {"npm": {"packages": ["single-file-cli"]}}}
|
||||
@@ -1,97 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Install hook for single-file binary.
|
||||
|
||||
Runs at crawl start to verify single-file (npm package) is available.
|
||||
Outputs JSONL for InstalledBinary and Machine config updates.
|
||||
Respects SINGLEFILE_BINARY env var for custom binary paths.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def find_singlefile() -> dict | None:
|
||||
"""Find single-file binary, respecting SINGLEFILE_BINARY env var."""
|
||||
try:
|
||||
from abx_pkg import Binary, NpmProvider, EnvProvider
|
||||
|
||||
# Check if user has configured a custom binary
|
||||
configured_binary = os.environ.get('SINGLEFILE_BINARY', '').strip()
|
||||
|
||||
if configured_binary:
|
||||
if '/' in configured_binary:
|
||||
bin_name = Path(configured_binary).name
|
||||
else:
|
||||
bin_name = configured_binary
|
||||
else:
|
||||
bin_name = 'single-file'
|
||||
|
||||
binary = Binary(name=bin_name, binproviders=[NpmProvider(), EnvProvider()])
|
||||
loaded = binary.load()
|
||||
if loaded and loaded.abspath:
|
||||
return {
|
||||
'name': bin_name,
|
||||
'abspath': str(loaded.abspath),
|
||||
'version': str(loaded.version) if loaded.version else None,
|
||||
'sha256': loaded.sha256 if hasattr(loaded, 'sha256') else None,
|
||||
'binprovider': loaded.binprovider.name if loaded.binprovider else 'env',
|
||||
}
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
# Determine binary name from config
|
||||
configured_binary = os.environ.get('SINGLEFILE_BINARY', '').strip()
|
||||
if configured_binary and '/' in configured_binary:
|
||||
bin_name = Path(configured_binary).name
|
||||
elif configured_binary:
|
||||
bin_name = configured_binary
|
||||
else:
|
||||
bin_name = 'single-file'
|
||||
|
||||
result = find_singlefile()
|
||||
|
||||
if result and result.get('abspath'):
|
||||
print(json.dumps({
|
||||
'type': 'InstalledBinary',
|
||||
'name': result['name'],
|
||||
'abspath': result['abspath'],
|
||||
'version': result['version'],
|
||||
'sha256': result['sha256'],
|
||||
'binprovider': result['binprovider'],
|
||||
}))
|
||||
|
||||
print(json.dumps({
|
||||
'type': 'Machine',
|
||||
'_method': 'update',
|
||||
'key': 'config/SINGLEFILE_BINARY',
|
||||
'value': result['abspath'],
|
||||
}))
|
||||
|
||||
if result['version']:
|
||||
print(json.dumps({
|
||||
'type': 'Machine',
|
||||
'_method': 'update',
|
||||
'key': 'config/SINGLEFILE_VERSION',
|
||||
'value': result['version'],
|
||||
}))
|
||||
|
||||
sys.exit(0)
|
||||
else:
|
||||
print(json.dumps({
|
||||
'type': 'Dependency',
|
||||
'bin_name': bin_name,
|
||||
'bin_providers': 'npm,env',
|
||||
}))
|
||||
print(f"{bin_name} binary not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -234,9 +234,9 @@ async function main() {
|
||||
// Install extension
|
||||
const extension = await installSinglefileExtension();
|
||||
|
||||
// Export extension metadata for chrome_session to load
|
||||
// Export extension metadata for chrome plugin to load
|
||||
if (extension) {
|
||||
// Write extension info to a cache file that chrome_session can read
|
||||
// Write extension info to a cache file that chrome plugin can read
|
||||
await fs.promises.mkdir(EXTENSIONS_DIR, { recursive: true });
|
||||
await fs.promises.writeFile(
|
||||
cacheFile,
|
||||
|
||||
@@ -28,10 +28,8 @@ Environment variables:
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import rich_click as click
|
||||
@@ -94,52 +92,11 @@ ALL_CHROME_BINARIES = (
|
||||
)
|
||||
|
||||
|
||||
def find_singlefile() -> str | None:
|
||||
"""Find SingleFile binary."""
|
||||
singlefile = get_env('SINGLEFILE_BINARY')
|
||||
if singlefile and os.path.isfile(singlefile):
|
||||
return singlefile
|
||||
|
||||
for name in ['single-file', 'singlefile']:
|
||||
binary = shutil.which(name)
|
||||
if binary:
|
||||
return binary
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def find_chrome() -> str | None:
|
||||
"""Find Chrome/Chromium binary."""
|
||||
chrome = get_env('CHROME_BINARY')
|
||||
if chrome and os.path.isfile(chrome):
|
||||
return chrome
|
||||
|
||||
for name in ALL_CHROME_BINARIES:
|
||||
if '/' in name:
|
||||
if os.path.isfile(name):
|
||||
return name
|
||||
else:
|
||||
binary = shutil.which(name)
|
||||
if binary:
|
||||
return binary
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_version(binary: str) -> str:
|
||||
"""Get SingleFile version."""
|
||||
try:
|
||||
result = subprocess.run([binary, '--version'], capture_output=True, text=True, timeout=10)
|
||||
return result.stdout.strip()[:64]
|
||||
except Exception:
|
||||
return ''
|
||||
|
||||
|
||||
CHROME_SESSION_DIR = '../chrome_session'
|
||||
CHROME_SESSION_DIR = '../chrome'
|
||||
|
||||
|
||||
def get_cdp_url() -> str | None:
|
||||
"""Get CDP URL from chrome_session if available."""
|
||||
"""Get CDP URL from chrome plugin if available."""
|
||||
cdp_file = Path(CHROME_SESSION_DIR) / 'cdp_url.txt'
|
||||
if cdp_file.exists():
|
||||
return cdp_file.read_text().strip()
|
||||
@@ -159,7 +116,7 @@ def save_singlefile(url: str, binary: str) -> tuple[bool, str | None, str]:
|
||||
"""
|
||||
Archive URL using SingleFile.
|
||||
|
||||
If a Chrome session exists (from chrome_session extractor), connects to it via CDP.
|
||||
If a Chrome session exists (from chrome plugin), connects to it via CDP.
|
||||
Otherwise launches a new Chrome instance.
|
||||
|
||||
Returns: (success, output_path, error_message)
|
||||
@@ -170,7 +127,7 @@ def save_singlefile(url: str, binary: str) -> tuple[bool, str | None, str]:
|
||||
check_ssl = get_env_bool('SINGLEFILE_CHECK_SSL_VALIDITY', get_env_bool('CHECK_SSL_VALIDITY', True))
|
||||
cookies_file = get_env('SINGLEFILE_COOKIES_FILE') or get_env('COOKIES_FILE', '')
|
||||
extra_args = get_env('SINGLEFILE_EXTRA_ARGS', '')
|
||||
chrome = find_chrome()
|
||||
chrome = get_env('CHROME_BINARY', '')
|
||||
|
||||
cmd = [binary]
|
||||
|
||||
@@ -234,13 +191,9 @@ def save_singlefile(url: str, binary: str) -> tuple[bool, str | None, str]:
|
||||
def main(url: str, snapshot_id: str):
|
||||
"""Archive a URL using SingleFile."""
|
||||
|
||||
start_ts = datetime.now(timezone.utc)
|
||||
version = ''
|
||||
output = None
|
||||
status = 'failed'
|
||||
error = ''
|
||||
binary = None
|
||||
cmd_str = ''
|
||||
|
||||
try:
|
||||
# Check if SingleFile is enabled
|
||||
@@ -255,33 +208,17 @@ def main(url: str, snapshot_id: str):
|
||||
print(json.dumps({'type': 'ArchiveResult', 'status': 'skipped', 'output_str': 'staticfile already exists'}))
|
||||
sys.exit(0)
|
||||
|
||||
# Find binary
|
||||
binary = find_singlefile()
|
||||
if not binary:
|
||||
print(f'ERROR: SingleFile binary not found', file=sys.stderr)
|
||||
print(f'DEPENDENCY_NEEDED={BIN_NAME}', file=sys.stderr)
|
||||
print(f'BIN_PROVIDERS={BIN_PROVIDERS}', file=sys.stderr)
|
||||
print(f'INSTALL_HINT=npm install -g single-file-cli', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
version = get_version(binary)
|
||||
cmd_str = f'{binary} {url} {OUTPUT_FILE}'
|
||||
# Get binary from environment
|
||||
binary = get_env('SINGLEFILE_BINARY', 'single-file')
|
||||
|
||||
# Run extraction
|
||||
success, output, error = save_singlefile(url, binary)
|
||||
status = 'succeeded' if success else 'failed'
|
||||
|
||||
if success and output:
|
||||
size = Path(output).stat().st_size
|
||||
print(f'SingleFile saved ({size} bytes)')
|
||||
|
||||
except Exception as e:
|
||||
error = f'{type(e).__name__}: {e}'
|
||||
status = 'failed'
|
||||
|
||||
# Calculate duration
|
||||
end_ts = datetime.now(timezone.utc)
|
||||
|
||||
if error:
|
||||
print(f'ERROR: {error}', file=sys.stderr)
|
||||
|
||||
@@ -291,10 +228,6 @@ def main(url: str, snapshot_id: str):
|
||||
'status': status,
|
||||
'output_str': output or error or '',
|
||||
}
|
||||
if binary:
|
||||
result['cmd'] = [binary, '--browser-headless', url, OUTPUT_FILE]
|
||||
if version:
|
||||
result['cmd_version'] = version
|
||||
print(json.dumps(result))
|
||||
|
||||
sys.exit(0 if status == 'succeeded' else 1)
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
"""
|
||||
Integration tests for singlefile plugin
|
||||
|
||||
Tests verify:
|
||||
1. on_Crawl hook validates and installs single-file
|
||||
2. Verify deps with abx-pkg
|
||||
3. Extraction works on https://example.com
|
||||
4. JSONL output is correct
|
||||
5. Filesystem output is valid HTML
|
||||
"""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
PLUGIN_DIR = Path(__file__).parent.parent
|
||||
PLUGINS_ROOT = PLUGIN_DIR.parent
|
||||
SINGLEFILE_HOOK = PLUGIN_DIR / "on_Snapshot__04_singlefile.js"
|
||||
CHROME_VALIDATE_HOOK = PLUGINS_ROOT / 'chrome_session' / 'on_Crawl__00_validate_chrome.py'
|
||||
NPM_PROVIDER_HOOK = PLUGINS_ROOT / 'npm' / 'on_Dependency__install_using_npm_provider.py'
|
||||
TEST_URL = "https://example.com"
|
||||
|
||||
|
||||
def test_hook_script_exists():
|
||||
"""Verify on_Snapshot hook exists."""
|
||||
assert SINGLEFILE_HOOK.exists(), f"Hook not found: {SINGLEFILE_HOOK}"
|
||||
|
||||
|
||||
def test_chrome_validation_and_install():
|
||||
"""Test chrome validation hook to install puppeteer-core if needed."""
|
||||
# Run chrome validation hook (from chrome_session plugin)
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(CHROME_VALIDATE_HOOK)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30
|
||||
)
|
||||
|
||||
# If exit 1, binary not found - need to install
|
||||
if result.returncode == 1:
|
||||
# Parse Dependency request from JSONL
|
||||
dependency_request = None
|
||||
for line in result.stdout.strip().split('\n'):
|
||||
if line.strip():
|
||||
try:
|
||||
record = json.loads(line)
|
||||
if record.get('type') == 'Dependency':
|
||||
dependency_request = record
|
||||
break
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
if dependency_request:
|
||||
bin_name = dependency_request['bin_name']
|
||||
bin_providers = dependency_request['bin_providers']
|
||||
|
||||
# Install via npm provider hook
|
||||
install_result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(NPM_PROVIDER_HOOK),
|
||||
'--dependency-id', 'test-dep-001',
|
||||
'--bin-name', bin_name,
|
||||
'--bin-providers', bin_providers
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=600
|
||||
)
|
||||
|
||||
assert install_result.returncode == 0, f"Install failed: {install_result.stderr}"
|
||||
|
||||
# Verify installation via JSONL output
|
||||
for line in install_result.stdout.strip().split('\n'):
|
||||
if line.strip():
|
||||
try:
|
||||
record = json.loads(line)
|
||||
if record.get('type') == 'InstalledBinary':
|
||||
assert record['name'] == bin_name
|
||||
assert record['abspath']
|
||||
break
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
else:
|
||||
# Binary already available, verify via JSONL output
|
||||
assert result.returncode == 0, f"Validation failed: {result.stderr}"
|
||||
|
||||
|
||||
def test_verify_deps_with_abx_pkg():
|
||||
"""Verify dependencies are available via abx-pkg after hook installation."""
|
||||
from abx_pkg import Binary, EnvProvider, BinProviderOverrides
|
||||
|
||||
EnvProvider.model_rebuild()
|
||||
|
||||
# Verify node is available (singlefile uses Chrome extension, needs Node)
|
||||
node_binary = Binary(name='node', binproviders=[EnvProvider()])
|
||||
node_loaded = node_binary.load()
|
||||
assert node_loaded and node_loaded.abspath, "Node.js required for singlefile plugin"
|
||||
|
||||
|
||||
def test_singlefile_hook_runs():
|
||||
"""Verify singlefile hook can be executed and completes."""
|
||||
# Prerequisites checked by earlier test
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmpdir = Path(tmpdir)
|
||||
|
||||
# Run singlefile extraction hook
|
||||
result = subprocess.run(
|
||||
['node', str(SINGLEFILE_HOOK), f'--url={TEST_URL}', '--snapshot-id=test789'],
|
||||
cwd=tmpdir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120
|
||||
)
|
||||
|
||||
# Hook should complete successfully (even if it just installs extension)
|
||||
assert result.returncode == 0, f"Hook execution failed: {result.stderr}"
|
||||
|
||||
# Verify extension installation happens
|
||||
assert 'SingleFile extension' in result.stdout or result.returncode == 0, "Should install extension or complete"
|
||||
@@ -212,7 +212,7 @@ describe('singlefile plugin', () => {
|
||||
assert.strictEqual(priority, 4);
|
||||
});
|
||||
|
||||
it('should run before chrome_session (priority 20)', () => {
|
||||
it('should run before chrome (priority 20)', () => {
|
||||
const extensionPriority = 4;
|
||||
const chromeSessionPriority = 20;
|
||||
|
||||
|
||||
@@ -1,12 +1,17 @@
|
||||
"""
|
||||
Unit tests for singlefile plugin
|
||||
Integration tests for singlefile plugin
|
||||
|
||||
Tests invoke the plugin hook as an external process and verify outputs/side effects.
|
||||
Tests verify:
|
||||
1. Hook script exists and has correct metadata
|
||||
2. Extension installation and caching works
|
||||
3. Chrome/node dependencies available
|
||||
4. Hook can be executed successfully
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
@@ -14,7 +19,11 @@ import pytest
|
||||
|
||||
|
||||
PLUGIN_DIR = Path(__file__).parent.parent
|
||||
PLUGINS_ROOT = PLUGIN_DIR.parent
|
||||
INSTALL_SCRIPT = PLUGIN_DIR / "on_Snapshot__04_singlefile.js"
|
||||
CHROME_INSTALL_HOOK = PLUGINS_ROOT / 'chrome' / 'on_Crawl__00_chrome_install.py'
|
||||
NPM_PROVIDER_HOOK = PLUGINS_ROOT / 'npm' / 'on_Binary__install_using_npm_provider.py'
|
||||
TEST_URL = "https://example.com"
|
||||
|
||||
|
||||
def test_install_script_exists():
|
||||
@@ -148,3 +157,102 @@ def test_output_directory_structure():
|
||||
assert "singlefile" in script_content.lower()
|
||||
# Should mention HTML output
|
||||
assert ".html" in script_content or "html" in script_content.lower()
|
||||
|
||||
|
||||
def test_chrome_validation_and_install():
|
||||
"""Test chrome install hook to install puppeteer-core if needed."""
|
||||
# Run chrome install hook (from chrome plugin)
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(CHROME_INSTALL_HOOK)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30
|
||||
)
|
||||
|
||||
# If exit 1, binary not found - need to install
|
||||
if result.returncode == 1:
|
||||
# Parse Dependency request from JSONL
|
||||
dependency_request = None
|
||||
for line in result.stdout.strip().split('\n'):
|
||||
if line.strip():
|
||||
try:
|
||||
record = json.loads(line)
|
||||
if record.get('type') == 'Dependency':
|
||||
dependency_request = record
|
||||
break
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
if dependency_request:
|
||||
bin_name = dependency_request['bin_name']
|
||||
bin_providers = dependency_request['bin_providers']
|
||||
|
||||
# Install via npm provider hook
|
||||
install_result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(NPM_PROVIDER_HOOK),
|
||||
'--dependency-id', 'test-dep-001',
|
||||
'--bin-name', bin_name,
|
||||
'--bin-providers', bin_providers
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=600
|
||||
)
|
||||
|
||||
assert install_result.returncode == 0, f"Install failed: {install_result.stderr}"
|
||||
|
||||
# Verify installation via JSONL output
|
||||
for line in install_result.stdout.strip().split('\n'):
|
||||
if line.strip():
|
||||
try:
|
||||
record = json.loads(line)
|
||||
if record.get('type') == 'Binary':
|
||||
assert record['name'] == bin_name
|
||||
assert record['abspath']
|
||||
break
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
else:
|
||||
# Binary already available, verify via JSONL output
|
||||
assert result.returncode == 0, f"Validation failed: {result.stderr}"
|
||||
|
||||
|
||||
def test_verify_deps_with_abx_pkg():
|
||||
"""Verify dependencies are available via abx-pkg after hook installation."""
|
||||
from abx_pkg import Binary, EnvProvider, BinProviderOverrides
|
||||
|
||||
EnvProvider.model_rebuild()
|
||||
|
||||
# Verify node is available (singlefile uses Chrome extension, needs Node)
|
||||
node_binary = Binary(name='node', binproviders=[EnvProvider()])
|
||||
node_loaded = node_binary.load()
|
||||
assert node_loaded and node_loaded.abspath, "Node.js required for singlefile plugin"
|
||||
|
||||
|
||||
def test_singlefile_hook_runs():
|
||||
"""Verify singlefile hook can be executed and completes."""
|
||||
# Prerequisites checked by earlier test
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
tmpdir = Path(tmpdir)
|
||||
|
||||
# Run singlefile extraction hook
|
||||
result = subprocess.run(
|
||||
['node', str(INSTALL_SCRIPT), f'--url={TEST_URL}', '--snapshot-id=test789'],
|
||||
cwd=tmpdir,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120
|
||||
)
|
||||
|
||||
# Hook should complete successfully (even if it just installs extension)
|
||||
assert result.returncode == 0, f"Hook execution failed: {result.stderr}"
|
||||
|
||||
# Verify extension installation happens
|
||||
assert 'SingleFile extension' in result.stdout or result.returncode == 0, "Should install extension or complete"
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main([__file__, '-v'])
|
||||
|
||||
Reference in New Issue
Block a user