alex/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-04-06 07:47:53 +10:00
Code Issues Packages Projects Releases Wiki Activity

add overrides options to binproviders

Browse Source
This commit is contained in:
Nick Sweeting
2025-12-26 20:16:58 -08:00
parent 6fdc52cc57
commit 9bc5d99488
13 changed files with 565 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
archivebox/plugins/apt/on_Dependency__install_using_apt_provider.py
View File

@@ -25,7 +25,8 @@ AptProvider.model_rebuild()
@click.option('--bin-name', required=True, help="Binary name to install")
@click.option('--bin-providers', default='*', help="Allowed providers (comma-separated)")
@click.option('--custom-cmd', default=None, help="Custom install command (overrides default)")
def main(dependency_id: str, bin_name: str, bin_providers: str, custom_cmd: str | None):
@click.option('--overrides', default=None, help="JSON-encoded overrides dict")
def main(dependency_id: str, bin_name: str, bin_providers: str, custom_cmd: str | None, overrides: str | None):
"""Install binary using apt package manager."""
# Check if apt provider is allowed
@@ -42,7 +43,16 @@ def main(dependency_id: str, bin_name: str, bin_providers: str, custom_cmd: str
click.echo(f"Installing {bin_name} via apt...", err=True)
try:
binary = Binary(name=bin_name, binproviders=[provider]).install()
# Parse overrides if provided
overrides_dict = None
if overrides:
try:
overrides_dict = json.loads(overrides)
click.echo(f"Using custom install overrides: {overrides_dict}", err=True)
except json.JSONDecodeError:
click.echo(f"Warning: Failed to parse overrides JSON: {overrides}", err=True)
binary = Binary(name=bin_name, binproviders=[provider], overrides=overrides_dict or {}).install()
except Exception as e:
click.echo(f"apt install failed: {e}", err=True)
sys.exit(1)

14
archivebox/plugins/brew/on_Dependency__install_using_brew_provider.py
View File

@@ -25,7 +25,8 @@ BrewProvider.model_rebuild()
@click.option('--bin-name', required=True, help="Binary name to install")
@click.option('--bin-providers', default='*', help="Allowed providers (comma-separated)")
@click.option('--custom-cmd', default=None, help="Custom install command")
def main(dependency_id: str, bin_name: str, bin_providers: str, custom_cmd: str | None):
@click.option('--overrides', default=None, help="JSON-encoded overrides dict")
def main(dependency_id: str, bin_name: str, bin_providers: str, custom_cmd: str | None, overrides: str | None):
"""Install binary using Homebrew."""
if bin_providers != '*' and 'brew' not in bin_providers.split(','):
@@ -41,7 +42,16 @@ def main(dependency_id: str, bin_name: str, bin_providers: str, custom_cmd: str
click.echo(f"Installing {bin_name} via brew...", err=True)
try:
binary = Binary(name=bin_name, binproviders=[provider]).install()
# Parse overrides if provided
overrides_dict = None
if overrides:
try:
overrides_dict = json.loads(overrides)
click.echo(f"Using custom install overrides: {overrides_dict}", err=True)
except json.JSONDecodeError:
click.echo(f"Warning: Failed to parse overrides JSON: {overrides}", err=True)
binary = Binary(name=bin_name, binproviders=[provider], overrides=overrides_dict or {}).install()
except Exception as e:
click.echo(f"brew install failed: {e}", err=True)
sys.exit(1)

43
archivebox/plugins/forumdl/on_Crawl__00_validate_forumdl.py
View File

@@ -15,42 +15,12 @@ import subprocess
from pathlib import Path
def get_binary_version(abspath: str, version_flag: str = '--version') -> str | None:
"""Get version string from binary."""
try:
result = subprocess.run(
[abspath, version_flag],
capture_output=True,
text=True,
timeout=5,
)
if result.returncode == 0 and result.stdout:
first_line = result.stdout.strip().split('\n')[0]
return first_line[:64]
except Exception:
pass
return None
def get_binary_hash(abspath: str) -> str | None:
"""Get SHA256 hash of binary."""
try:
with open(abspath, 'rb') as f:
return hashlib.sha256(f.read()).hexdigest()
except Exception:
return None
def find_forumdl() -> dict | None:
"""Find forum-dl binary."""
try:
from abx_pkg import Binary, PipProvider, EnvProvider
class ForumdlBinary(Binary):
name: str = 'forum-dl'
binproviders_supported = [PipProvider(), EnvProvider()]
binary = ForumdlBinary()
binary = Binary(name='forum-dl', binproviders=[PipProvider(), EnvProvider()])
loaded = binary.load()
if loaded and loaded.abspath:
return {
@@ -86,7 +56,7 @@ def main():
missing_deps = []
# Emit results for forum-dl
if forumdl_result and forumdl_result.get('abspath'):
if forumdl_result and forumdl_result.get('abspath') and forumdl_result.get('version'):
print(json.dumps({
'type': 'InstalledBinary',
'name': forumdl_result['name'],
@@ -111,10 +81,19 @@ def main():
'value': forumdl_result['version'],
}))
else:
# forum-dl has cchardet dependency that doesn't compile on Python 3.14+
# Provide overrides to install with chardet instead
print(json.dumps({
'type': 'Dependency',
'bin_name': 'forum-dl',
'bin_providers': 'pip,env',
'overrides': {
'pip': {
'packages': ['--no-deps', 'forum-dl', 'chardet', 'pydantic', 'beautifulsoup4', 'lxml',
'requests', 'urllib3', 'tenacity', 'python-dateutil',
'html2text', 'warcio']
}
}
}))
missing_deps.append('forum-dl')

2
archivebox/plugins/forumdl/on_Snapshot__53_forumdl.py
View File

@@ -137,6 +137,8 @@ def save_forum(url: str, binary: str) -> tuple[bool, str | None, str]:
return True, None, '' # Not a forum site - success, no output
if 'no content' in stderr_lower:
return True, None, '' # No forum found - success, no output
if 'extractornotfounderror' in stderr_lower:
return True, None, '' # No forum extractor for this URL - success, no output
if result.returncode == 0:
return True, None, '' # forum-dl exited cleanly, just no forum - success

106
archivebox/plugins/forumdl/tests/test_forumdl.py
View File

@@ -15,6 +15,7 @@ import json
import subprocess
import sys
import tempfile
import uuid
from pathlib import Path
import pytest
@@ -24,6 +25,75 @@ FORUMDL_HOOK = PLUGIN_DIR / 'on_Snapshot__53_forumdl.py'
FORUMDL_VALIDATE_HOOK = PLUGIN_DIR / 'on_Crawl__00_validate_forumdl.py'
TEST_URL = 'https://example.com'
# Module-level cache for installed binary path
_forumdl_binary_path = None
def get_forumdl_binary_path():
"""Get the installed forum-dl binary path from cache or by running validation/installation."""
global _forumdl_binary_path
if _forumdl_binary_path:
return _forumdl_binary_path
# Run validation hook to find or install binary
result = subprocess.run(
[sys.executable, str(FORUMDL_VALIDATE_HOOK)],
capture_output=True,
text=True,
timeout=300
)
# Check if binary was found
for line in result.stdout.strip().split('\n'):
if line.strip():
try:
record = json.loads(line)
if record.get('type') == 'InstalledBinary' and record.get('name') == 'forum-dl':
_forumdl_binary_path = record.get('abspath')
return _forumdl_binary_path
elif record.get('type') == 'Dependency' and record.get('bin_name') == 'forum-dl':
# Need to install via pip hook
pip_hook = PLUGINS_ROOT / 'pip' / 'on_Dependency__install_using_pip_provider.py'
dependency_id = str(uuid.uuid4())
# Build command with overrides if present
cmd = [
sys.executable, str(pip_hook),
'--dependency-id', dependency_id,
'--bin-name', record['bin_name']
]
if 'overrides' in record:
cmd.extend(['--overrides', json.dumps(record['overrides'])])
install_result = subprocess.run(
cmd,
capture_output=True,
text=True,
timeout=300
)
# Parse InstalledBinary from pip installation
for install_line in install_result.stdout.strip().split('\n'):
if install_line.strip():
try:
install_record = json.loads(install_line)
if install_record.get('type') == 'InstalledBinary' and install_record.get('name') == 'forum-dl':
_forumdl_binary_path = install_record.get('abspath')
return _forumdl_binary_path
except json.JSONDecodeError:
pass
# Installation failed - print debug info
if not _forumdl_binary_path:
print(f"\n=== forum-dl installation failed ===", file=sys.stderr)
print(f"stdout: {install_result.stdout}", file=sys.stderr)
print(f"stderr: {install_result.stderr}", file=sys.stderr)
print(f"returncode: {install_result.returncode}", file=sys.stderr)
return None
except json.JSONDecodeError:
pass
return None
def test_hook_script_exists():
"""Verify on_Snapshot hook exists."""
assert FORUMDL_HOOK.exists(), f"Hook not found: {FORUMDL_HOOK}"
@@ -64,38 +134,40 @@ def test_forumdl_validate_hook():
def test_verify_deps_with_abx_pkg():
"""Verify forum-dl is available via abx-pkg."""
from abx_pkg import Binary, PipProvider, EnvProvider, BinProviderOverrides
missing_binaries = []
# Verify forum-dl is available
forumdl_binary = Binary(name='forum-dl', binproviders=[PipProvider(), EnvProvider()])
forumdl_loaded = forumdl_binary.load()
if not (forumdl_loaded and forumdl_loaded.abspath):
missing_binaries.append('forum-dl')
if missing_binaries:
pytest.skip(f"Binaries not available: {', '.join(missing_binaries)} - Dependency records should have been emitted")
"""Verify forum-dl is installed by calling the REAL validation and installation hooks."""
binary_path = get_forumdl_binary_path()
assert binary_path, (
"forum-dl must be installed successfully via validation hook and pip provider. "
"NOTE: forum-dl has a dependency on cchardet which does not compile on Python 3.14+ "
"due to removed longintrepr.h header. This is a known compatibility issue with forum-dl."
)
assert Path(binary_path).is_file(), f"Binary path must be a valid file: {binary_path}"
def test_handles_non_forum_url():
"""Test that forum-dl extractor handles non-forum URLs gracefully via hook."""
# Prerequisites checked by earlier test
import os
binary_path = get_forumdl_binary_path()
assert binary_path, "Binary must be installed for this test"
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir = Path(tmpdir)
env = os.environ.copy()
env['FORUMDL_BINARY'] = binary_path
# Run forum-dl extraction hook on non-forum URL
result = subprocess.run(
[sys.executable, str(FORUMDL_HOOK), '--url', 'https://example.com', '--snapshot-id', 'test789'],
cwd=tmpdir,
capture_output=True,
text=True,
env=env,
timeout=60
)
# Should exit 0 even for non-forum URL
# Should exit 0 even for non-forum URL (graceful handling)
assert result.returncode == 0, f"Should handle non-forum URL gracefully: {result.stderr}"
# Verify JSONL output
@@ -138,8 +210,12 @@ def test_config_timeout():
"""Test that FORUMDL_TIMEOUT config is respected."""
import os
binary_path = get_forumdl_binary_path()
assert binary_path, "Binary must be installed for this test"
with tempfile.TemporaryDirectory() as tmpdir:
env = os.environ.copy()
env['FORUMDL_BINARY'] = binary_path
env['FORUMDL_TIMEOUT'] = '5'
result = subprocess.run(

65
archivebox/plugins/media/on_Crawl__00_validate_ytdlp.py
View File

@@ -15,42 +15,12 @@ import subprocess
from pathlib import Path
def get_binary_version(abspath: str, version_flag: str = '--version') -> str | None:
"""Get version string from binary."""
try:
result = subprocess.run(
[abspath, version_flag],
capture_output=True,
text=True,
timeout=5,
)
if result.returncode == 0 and result.stdout:
first_line = result.stdout.strip().split('\n')[0]
return first_line[:64]
except Exception:
pass
return None
def get_binary_hash(abspath: str) -> str | None:
"""Get SHA256 hash of binary."""
try:
with open(abspath, 'rb') as f:
return hashlib.sha256(f.read()).hexdigest()
except Exception:
return None
def find_ytdlp() -> dict | None:
"""Find yt-dlp binary."""
try:
from abx_pkg import Binary, PipProvider, EnvProvider
from abx_pkg import Binary, PipProvider, BrewProvider, AptProvider, EnvProvider
class YtdlpBinary(Binary):
name: str = 'yt-dlp'
binproviders_supported = [PipProvider(), EnvProvider()]
binary = YtdlpBinary()
binary = Binary(name='yt-dlp', binproviders=[PipProvider(), BrewProvider(), AptProvider(), EnvProvider()])
loaded = binary.load()
if loaded and loaded.abspath:
return {
@@ -71,8 +41,8 @@ def find_ytdlp() -> dict | None:
return {
'name': 'yt-dlp',
'abspath': abspath,
'version': get_binary_version(abspath),
'sha256': get_binary_hash(abspath),
'version': None,
'sha256': None,
'binprovider': 'env',
}
@@ -84,12 +54,7 @@ def find_node() -> dict | None:
try:
from abx_pkg import Binary, AptProvider, BrewProvider, EnvProvider
class NodeBinary(Binary):
name: str = 'node'
binproviders_supported = [AptProvider(), BrewProvider(), EnvProvider()]
overrides: dict = {'apt': {'packages': ['nodejs']}}
binary = NodeBinary()
binary = Binary(name='node', binproviders=[AptProvider(), BrewProvider(), EnvProvider()])
loaded = binary.load()
if loaded and loaded.abspath:
return {
@@ -110,8 +75,8 @@ def find_node() -> dict | None:
return {
'name': 'node',
'abspath': abspath,
'version': get_binary_version(abspath),
'sha256': get_binary_hash(abspath),
'version': None,
'sha256': None,
'binprovider': 'env',
}
@@ -123,11 +88,7 @@ def find_ffmpeg() -> dict | None:
try:
from abx_pkg import Binary, AptProvider, BrewProvider, EnvProvider
class FfmpegBinary(Binary):
name: str = 'ffmpeg'
binproviders_supported = [AptProvider(), BrewProvider(), EnvProvider()]
binary = FfmpegBinary()
binary = Binary(name='ffmpeg', binproviders=[AptProvider(), BrewProvider(), EnvProvider()])
loaded = binary.load()
if loaded and loaded.abspath:
return {
@@ -148,8 +109,8 @@ def find_ffmpeg() -> dict | None:
return {
'name': 'ffmpeg',
'abspath': abspath,
'version': get_binary_version(abspath),
'sha256': get_binary_hash(abspath),
'version': None,
'sha256': None,
'binprovider': 'env',
}
@@ -197,7 +158,7 @@ def main():
print(json.dumps({
'type': 'Dependency',
'bin_name': 'yt-dlp',
'bin_providers': 'pip,env',
'bin_providers': 'pip,brew,apt,env',
}))
missing_deps.append('yt-dlp')
@@ -227,10 +188,14 @@ def main():
'value': node_result['version'],
}))
else:
# node is installed as 'nodejs' package on apt
print(json.dumps({
'type': 'Dependency',
'bin_name': 'node',
'bin_providers': 'apt,brew,env',
'overrides': {
'apt': {'packages': ['nodejs']}
}
}))
missing_deps.append('node')

41
archivebox/plugins/mercury/on_Crawl__00_validate_mercury.py
View File

@@ -15,43 +15,12 @@ import subprocess
from pathlib import Path
def get_binary_version(abspath: str) -> str | None:
"""Get version string from binary."""
try:
result = subprocess.run(
[abspath, '--version'],
capture_output=True,
text=True,
timeout=5,
)
if result.returncode == 0 and result.stdout:
first_line = result.stdout.strip().split('\n')[0]
return first_line[:64]
except Exception:
pass
return None
def get_binary_hash(abspath: str) -> str | None:
"""Get SHA256 hash of binary."""
try:
with open(abspath, 'rb') as f:
return hashlib.sha256(f.read()).hexdigest()
except Exception:
return None
def find_mercury() -> dict | None:
"""Find postlight-parser binary."""
try:
from abx_pkg import Binary, NpmProvider, EnvProvider
class MercuryBinary(Binary):
name: str = 'postlight-parser'
binproviders_supported = [NpmProvider(), EnvProvider()]
overrides: dict = {'npm': {'packages': ['@postlight/parser']}}
binary = MercuryBinary()
binary = Binary(name='postlight-parser', binproviders=[NpmProvider(), EnvProvider()])
loaded = binary.load()
if loaded and loaded.abspath:
return {
@@ -72,8 +41,8 @@ def find_mercury() -> dict | None:
return {
'name': 'postlight-parser',
'abspath': abspath,
'version': get_binary_version(abspath),
'sha256': get_binary_hash(abspath),
'version': None,
'sha256': None,
'binprovider': 'env',
}
@@ -110,10 +79,14 @@ def main():
sys.exit(0)
else:
# postlight-parser is installed as @postlight/parser in npm
print(json.dumps({
'type': 'Dependency',
'bin_name': 'postlight-parser',
'bin_providers': 'npm,env',
'overrides': {
'npm': {'packages': ['@postlight/parser']}
}
}))
print(f"postlight-parser binary not found", file=sys.stderr)
sys.exit(1)

14
archivebox/plugins/npm/on_Dependency__install_using_npm_provider.py
View File

@@ -25,7 +25,8 @@ NpmProvider.model_rebuild()
@click.option('--bin-name', required=True, help="Binary name to install")
@click.option('--bin-providers', default='*', help="Allowed providers (comma-separated)")
@click.option('--custom-cmd', default=None, help="Custom install command")
def main(dependency_id: str, bin_name: str, bin_providers: str, custom_cmd: str | None):
@click.option('--overrides', default=None, help="JSON-encoded overrides dict")
def main(dependency_id: str, bin_name: str, bin_providers: str, custom_cmd: str | None, overrides: str | None):
"""Install binary using npm."""
if bin_providers != '*' and 'npm' not in bin_providers.split(','):
@@ -41,7 +42,16 @@ def main(dependency_id: str, bin_name: str, bin_providers: str, custom_cmd: str
click.echo(f"Installing {bin_name} via npm...", err=True)
try:
binary = Binary(name=bin_name, binproviders=[provider]).install()
# Parse overrides if provided
overrides_dict = None
if overrides:
try:
overrides_dict = json.loads(overrides)
click.echo(f"Using custom install overrides: {overrides_dict}", err=True)
except json.JSONDecodeError:
click.echo(f"Warning: Failed to parse overrides JSON: {overrides}", err=True)
binary = Binary(name=bin_name, binproviders=[provider], overrides=overrides_dict or {}).install()
except Exception as e:
click.echo(f"npm install failed: {e}", err=True)
sys.exit(1)

92
archivebox/plugins/papersdl/tests/test_papersdl.py
View File

@@ -15,6 +15,7 @@ import json
import subprocess
import sys
import tempfile
import uuid
from pathlib import Path
import pytest
@@ -24,6 +25,67 @@ PAPERSDL_HOOK = PLUGIN_DIR / 'on_Snapshot__54_papersdl.py'
PAPERSDL_VALIDATE_HOOK = PLUGIN_DIR / 'on_Crawl__00_validate_papersdl.py'
TEST_URL = 'https://example.com'
# Module-level cache for installed binary path
_papersdl_binary_path = None
def get_papersdl_binary_path():
"""Get the installed papers-dl binary path from cache or by running validation/installation."""
global _papersdl_binary_path
if _papersdl_binary_path:
return _papersdl_binary_path
# Run validation hook to find or install binary
result = subprocess.run(
[sys.executable, str(PAPERSDL_VALIDATE_HOOK)],
capture_output=True,
text=True,
timeout=300
)
# Check if binary was found
for line in result.stdout.strip().split('\n'):
if line.strip():
try:
record = json.loads(line)
if record.get('type') == 'InstalledBinary' and record.get('name') == 'papers-dl':
_papersdl_binary_path = record.get('abspath')
return _papersdl_binary_path
elif record.get('type') == 'Dependency' and record.get('bin_name') == 'papers-dl':
# Need to install via pip hook
pip_hook = PLUGINS_ROOT / 'pip' / 'on_Dependency__install_using_pip_provider.py'
dependency_id = str(uuid.uuid4())
# Build command with overrides if present
cmd = [
sys.executable, str(pip_hook),
'--dependency-id', dependency_id,
'--bin-name', record['bin_name']
]
if 'overrides' in record:
cmd.extend(['--overrides', json.dumps(record['overrides'])])
install_result = subprocess.run(
cmd,
capture_output=True,
text=True,
timeout=300
)
# Parse InstalledBinary from pip installation
for install_line in install_result.stdout.strip().split('\n'):
if install_line.strip():
try:
install_record = json.loads(install_line)
if install_record.get('type') == 'InstalledBinary' and install_record.get('name') == 'papers-dl':
_papersdl_binary_path = install_record.get('abspath')
return _papersdl_binary_path
except json.JSONDecodeError:
pass
except json.JSONDecodeError:
pass
return None
def test_hook_script_exists():
"""Verify on_Snapshot hook exists."""
assert PAPERSDL_HOOK.exists(), f"Hook not found: {PAPERSDL_HOOK}"
@@ -64,34 +126,32 @@ def test_papersdl_validate_hook():
def test_verify_deps_with_abx_pkg():
"""Verify papers-dl is available via abx-pkg."""
from abx_pkg import Binary, PipProvider, EnvProvider, BinProviderOverrides
missing_binaries = []
# Verify papers-dl is available
papersdl_binary = Binary(name='papers-dl', binproviders=[PipProvider(), EnvProvider()])
papersdl_loaded = papersdl_binary.load()
if not (papersdl_loaded and papersdl_loaded.abspath):
missing_binaries.append('papers-dl')
if missing_binaries:
pytest.skip(f"Binaries not available: {', '.join(missing_binaries)} - Dependency records should have been emitted")
"""Verify papers-dl is installed by calling the REAL validation and installation hooks."""
binary_path = get_papersdl_binary_path()
assert binary_path, "papers-dl must be installed successfully via validation hook and pip provider"
assert Path(binary_path).is_file(), f"Binary path must be a valid file: {binary_path}"
def test_handles_non_paper_url():
"""Test that papers-dl extractor handles non-paper URLs gracefully via hook."""
# Prerequisites checked by earlier test
import os
binary_path = get_papersdl_binary_path()
assert binary_path, "Binary must be installed for this test"
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir = Path(tmpdir)
env = os.environ.copy()
env['PAPERSDL_BINARY'] = binary_path
# Run papers-dl extraction hook on non-paper URL
result = subprocess.run(
[sys.executable, str(PAPERSDL_HOOK), '--url', 'https://example.com', '--snapshot-id', 'test789'],
cwd=tmpdir,
capture_output=True,
text=True,
env=env,
timeout=60
)
@@ -138,8 +198,12 @@ def test_config_timeout():
"""Test that PAPERSDL_TIMEOUT config is respected."""
import os
binary_path = get_papersdl_binary_path()
assert binary_path, "Binary must be installed for this test"
with tempfile.TemporaryDirectory() as tmpdir:
env = os.environ.copy()
env['PAPERSDL_BINARY'] = binary_path
env['PAPERSDL_TIMEOUT'] = '5'
result = subprocess.run(

14
archivebox/plugins/pip/on_Dependency__install_using_pip_provider.py
View File

@@ -25,7 +25,8 @@ PipProvider.model_rebuild()
@click.option('--bin-name', required=True, help="Binary name to install")
@click.option('--bin-providers', default='*', help="Allowed providers (comma-separated)")
@click.option('--custom-cmd', default=None, help="Custom install command")
def main(dependency_id: str, bin_name: str, bin_providers: str, custom_cmd: str | None):
@click.option('--overrides', default=None, help="JSON-encoded overrides dict")
def main(dependency_id: str, bin_name: str, bin_providers: str, custom_cmd: str | None, overrides: str | None):
"""Install binary using pip."""
if bin_providers != '*' and 'pip' not in bin_providers.split(','):
@@ -41,7 +42,16 @@ def main(dependency_id: str, bin_name: str, bin_providers: str, custom_cmd: str
click.echo(f"Installing {bin_name} via pip...", err=True)
try:
binary = Binary(name=bin_name, binproviders=[provider]).install()
# Parse overrides if provided
overrides_dict = None
if overrides:
try:
overrides_dict = json.loads(overrides)
click.echo(f"Using custom install overrides: {overrides_dict}", err=True)
except json.JSONDecodeError:
click.echo(f"Warning: Failed to parse overrides JSON: {overrides}", err=True)
binary = Binary(name=bin_name, binproviders=[provider], overrides=overrides_dict or {}).install()
except Exception as e:
click.echo(f"pip install failed: {e}", err=True)
sys.exit(1)

41
archivebox/plugins/readability/on_Crawl__00_validate_readability.py
View File

@@ -15,43 +15,12 @@ import subprocess
from pathlib import Path
def get_binary_version(abspath: str) -> str | None:
"""Get version string from binary."""
try:
result = subprocess.run(
[abspath, '--version'],
capture_output=True,
text=True,
timeout=5,
)
if result.returncode == 0 and result.stdout:
first_line = result.stdout.strip().split('\n')[0]
return first_line[:64]
except Exception:
pass
return None
def get_binary_hash(abspath: str) -> str | None:
"""Get SHA256 hash of binary."""
try:
with open(abspath, 'rb') as f:
return hashlib.sha256(f.read()).hexdigest()
except Exception:
return None
def find_readability() -> dict | None:
"""Find readability-extractor binary."""
try:
from abx_pkg import Binary, NpmProvider, EnvProvider
class ReadabilityBinary(Binary):
name: str = 'readability-extractor'
binproviders_supported = [NpmProvider(), EnvProvider()]
overrides: dict = {'npm': {'packages': ['github:ArchiveBox/readability-extractor']}}
binary = ReadabilityBinary()
binary = Binary(name='readability-extractor', binproviders=[NpmProvider(), EnvProvider()])
loaded = binary.load()
if loaded and loaded.abspath:
return {
@@ -72,8 +41,8 @@ def find_readability() -> dict | None:
return {
'name': 'readability-extractor',
'abspath': abspath,
'version': get_binary_version(abspath),
'sha256': get_binary_hash(abspath),
'version': None,
'sha256': None,
'binprovider': 'env',
}
@@ -110,10 +79,14 @@ def main():
sys.exit(0)
else:
# readability-extractor is installed from GitHub
print(json.dumps({
'type': 'Dependency',
'bin_name': 'readability-extractor',
'bin_providers': 'npm,env',
'overrides': {
'npm': {'packages': ['github:ArchiveBox/readability-extractor']}
}
}))
print(f"readability-extractor binary not found", file=sys.stderr)
sys.exit(1)