alex/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-04-05 23:37:58 +10:00
Code Issues Packages Projects Releases Wiki Activity

way better plugin hooks system wip

Browse Source
This commit is contained in:
Nick Sweeting
2025-12-28 03:39:59 -08:00
parent a38624a4dd
commit 50e527ec65
156 changed files with 10275 additions and 7149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
archivebox/plugins/git/binaries.jsonl Normal file
View File

@@ -0,0 +1 @@
{"type": "Binary", "name": "git", "binproviders": "apt,brew,env"}

97
archivebox/plugins/git/on_Crawl__00_install_git.py
View File

@@ -1,97 +0,0 @@
#!/usr/bin/env python3
"""
Install hook for git binary.
Runs at crawl start to verify git is available.
Outputs JSONL for InstalledBinary and Machine config updates.
Respects GIT_BINARY env var for custom binary paths.
"""
import os
import sys
import json
from pathlib import Path
def find_git() -> dict | None:
"""Find git binary, respecting GIT_BINARY env var."""
try:
from abx_pkg import Binary, EnvProvider
# Check if user has configured a custom binary
configured_binary = os.environ.get('GIT_BINARY', '').strip()
if configured_binary:
if '/' in configured_binary:
bin_name = Path(configured_binary).name
else:
bin_name = configured_binary
else:
bin_name = 'git'
binary = Binary(name=bin_name, binproviders=[EnvProvider()])
loaded = binary.load()
if loaded and loaded.abspath:
return {
'name': bin_name,
'abspath': str(loaded.abspath),
'version': str(loaded.version) if loaded.version else None,
'sha256': loaded.sha256 if hasattr(loaded, 'sha256') else None,
'binprovider': loaded.binprovider.name if loaded.binprovider else 'env',
}
except Exception:
pass
return None
def main():
# Determine binary name from config
configured_binary = os.environ.get('GIT_BINARY', '').strip()
if configured_binary and '/' in configured_binary:
bin_name = Path(configured_binary).name
elif configured_binary:
bin_name = configured_binary
else:
bin_name = 'git'
result = find_git()
if result and result.get('abspath'):
print(json.dumps({
'type': 'InstalledBinary',
'name': result['name'],
'abspath': result['abspath'],
'version': result['version'],
'sha256': result['sha256'],
'binprovider': result['binprovider'],
}))
print(json.dumps({
'type': 'Machine',
'_method': 'update',
'key': 'config/GIT_BINARY',
'value': result['abspath'],
}))
if result['version']:
print(json.dumps({
'type': 'Machine',
'_method': 'update',
'key': 'config/GIT_VERSION',
'value': result['version'],
}))
sys.exit(0)
else:
print(json.dumps({
'type': 'Dependency',
'bin_name': bin_name,
'bin_providers': 'apt,brew,env',
}))
print(f"{bin_name} binary not found", file=sys.stderr)
sys.exit(1)
if __name__ == '__main__':
main()

64
archivebox/plugins/git/on_Snapshot__12_git.py
View File

@@ -7,16 +7,17 @@ Output: Clones repository to $PWD/repo
Environment variables:
GIT_BINARY: Path to git binary
TIMEOUT: Timeout in seconds (default: 120)
GIT_TIMEOUT: Timeout in seconds (default: 120)
GIT_ARGS: Extra arguments for git clone (space-separated)
# Fallback to ARCHIVING_CONFIG values if GIT_* not set:
TIMEOUT: Fallback timeout
"""
import json
import os
import shutil
import subprocess
import sys
from datetime import datetime, timezone
from pathlib import Path
import rich_click as click
@@ -53,31 +54,13 @@ def is_git_url(url: str) -> bool:
return any(p in url.lower() for p in git_patterns)
def find_git() -> str | None:
"""Find git binary."""
git = get_env('GIT_BINARY')
if git and os.path.isfile(git):
return git
return shutil.which('git')
def get_version(binary: str) -> str:
"""Get git version."""
try:
result = subprocess.run([binary, '--version'], capture_output=True, text=True, timeout=10)
return result.stdout.strip()[:64]
except Exception:
return ''
def clone_git(url: str, binary: str) -> tuple[bool, str | None, str]:
"""
Clone git repository.
Returns: (success, output_path, error_message)
"""
timeout = get_env_int('TIMEOUT', 120)
timeout = get_env_int('GIT_TIMEOUT') or get_env_int('TIMEOUT', 120)
extra_args = get_env('GIT_ARGS')
cmd = [
@@ -113,49 +96,32 @@ def clone_git(url: str, binary: str) -> tuple[bool, str | None, str]:
def main(url: str, snapshot_id: str):
"""Clone a git repository from a URL."""
start_ts = datetime.now(timezone.utc)
version = ''
output = None
status = 'failed'
error = ''
binary = None
try:
# Check if URL looks like a git repo
if not is_git_url(url):
print(f'Skipping git clone for non-git URL: {url}')
status = 'skipped'
end_ts = datetime.now(timezone.utc)
print(f'START_TS={start_ts.isoformat()}')
print(f'END_TS={end_ts.isoformat()}')
print(f'STATUS={status}')
print(f'RESULT_JSON={json.dumps({"extractor": EXTRACTOR_NAME, "status": status, "url": url})}')
print(f'Skipping git clone for non-git URL: {url}', file=sys.stderr)
print(json.dumps({
'type': 'ArchiveResult',
'status': 'skipped',
'output_str': 'Not a git URL',
}))
sys.exit(0)
# Find binary
binary = find_git()
if not binary:
print(f'ERROR: git binary not found', file=sys.stderr)
print(f'DEPENDENCY_NEEDED={BIN_NAME}', file=sys.stderr)
print(f'BIN_PROVIDERS={BIN_PROVIDERS}', file=sys.stderr)
sys.exit(1)
version = get_version(binary)
# Get binary from environment
binary = get_env('GIT_BINARY', 'git')
# Run extraction
success, output, error = clone_git(url, binary)
status = 'succeeded' if success else 'failed'
if success:
print(f'git clone completed')
except Exception as e:
error = f'{type(e).__name__}: {e}'
status = 'failed'
# Calculate duration
end_ts = datetime.now(timezone.utc)
if error:
print(f'ERROR: {error}', file=sys.stderr)
@@ -165,10 +131,6 @@ def main(url: str, snapshot_id: str):
'status': status,
'output_str': output or error or '',
}
if binary:
result['cmd'] = [binary, 'clone', '--depth=1', '--recursive', url, OUTPUT_DIR]
if version:
result['cmd_version'] = version
print(json.dumps(result))
sys.exit(0 if status == 'succeeded' else 1)

34
archivebox/plugins/git/tests/test_git.py
View File

@@ -17,16 +17,16 @@ import pytest
PLUGIN_DIR = Path(__file__).parent.parent
GIT_HOOK = PLUGIN_DIR / 'on_Snapshot__12_git.py'
GIT_VALIDATE_HOOK = PLUGIN_DIR / 'on_Crawl__00_validate_git.py'
GIT_INSTALL_HOOK = PLUGIN_DIR / 'on_Crawl__00_install_git.py'
TEST_URL = 'https://github.com/example/repo.git'
def test_hook_script_exists():
assert GIT_HOOK.exists()
def test_git_validate_hook():
"""Test git validate hook checks for git binary."""
def test_git_install_hook():
"""Test git install hook checks for git binary."""
result = subprocess.run(
[sys.executable, str(GIT_VALIDATE_HOOK)],
[sys.executable, str(GIT_INSTALL_HOOK)],
capture_output=True,
text=True,
timeout=30
@@ -34,20 +34,20 @@ def test_git_validate_hook():
# Hook exits 0 if binary found, 1 if not found (with Dependency record)
if result.returncode == 0:
# Binary found - verify InstalledBinary JSONL output
# Binary found - verify Binary JSONL output
found_binary = False
for line in result.stdout.strip().split('\n'):
if line.strip():
try:
record = json.loads(line)
if record.get('type') == 'InstalledBinary':
if record.get('type') == 'Binary':
assert record['name'] == 'git'
assert record['abspath']
found_binary = True
break
except json.JSONDecodeError:
pass
assert found_binary, "Should output InstalledBinary record when binary found"
assert found_binary, "Should output Binary record when binary found"
else:
# Binary not found - verify Dependency JSONL output
found_dependency = False
@@ -90,7 +90,7 @@ def test_reports_missing_git():
def test_handles_non_git_url():
if not shutil.which('git'):
pytest.skip("git not installed")
with tempfile.TemporaryDirectory() as tmpdir:
result = subprocess.run(
[sys.executable, str(GIT_HOOK), '--url', 'https://example.com', '--snapshot-id', 'test789'],
@@ -98,7 +98,23 @@ def test_handles_non_git_url():
)
# Should fail or skip for non-git URL
assert result.returncode in (0, 1)
assert 'STATUS=' in result.stdout
# Parse clean JSONL output
result_json = None
for line in result.stdout.strip().split('\n'):
line = line.strip()
if line.startswith('{'):
try:
record = json.loads(line)
if record.get('type') == 'ArchiveResult':
result_json = record
break
except json.JSONDecodeError:
pass
if result_json:
# Should report failure or skip for non-git URL
assert result_json['status'] in ['failed', 'skipped'], f"Should fail or skip: {result_json}"
if __name__ == '__main__':
pytest.main([__file__, '-v'])