mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-04-05 15:27:53 +10:00
remove huey
This commit is contained in:
Nick Sweeting
@@ -1,67 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Install yt-dlp if not already available.
|
||||
|
||||
Runs at crawl start to ensure yt-dlp is installed.
|
||||
Outputs JSONL for InstalledBinary.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
from abx_pkg import Binary, PipProvider, EnvProvider, BinProviderOverrides
|
||||
|
||||
PipProvider.model_rebuild()
|
||||
EnvProvider.model_rebuild()
|
||||
|
||||
# yt-dlp binary and package have same name
|
||||
ytdlp_binary = Binary(
|
||||
name='yt-dlp',
|
||||
binproviders=[PipProvider(), EnvProvider()]
|
||||
)
|
||||
|
||||
# Try to load, install if not found
|
||||
try:
|
||||
loaded = ytdlp_binary.load()
|
||||
if not loaded or not loaded.abspath:
|
||||
raise Exception("Not loaded")
|
||||
except Exception:
|
||||
# Install via pip
|
||||
loaded = ytdlp_binary.install()
|
||||
|
||||
if loaded and loaded.abspath:
|
||||
# Output InstalledBinary JSONL
|
||||
print(json.dumps({
|
||||
'type': 'InstalledBinary',
|
||||
'name': 'yt-dlp',
|
||||
'abspath': str(loaded.abspath),
|
||||
'version': str(loaded.version) if loaded.version else None,
|
||||
'sha256': loaded.sha256,
|
||||
'binprovider': loaded.loaded_binprovider.name if loaded.loaded_binprovider else 'unknown',
|
||||
}))
|
||||
sys.exit(0)
|
||||
else:
|
||||
print(json.dumps({
|
||||
'type': 'Dependency',
|
||||
'bin_name': 'yt-dlp',
|
||||
'bin_providers': 'pip,brew,env',
|
||||
}))
|
||||
print("Failed to install yt-dlp", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
except Exception as e:
|
||||
print(json.dumps({
|
||||
'type': 'Dependency',
|
||||
'bin_name': 'yt-dlp',
|
||||
'bin_providers': 'pip,brew,env',
|
||||
}))
|
||||
print(f"Error installing yt-dlp: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
278
archivebox/plugins/media/on_Crawl__00_validate_ytdlp.py
Executable file
278
archivebox/plugins/media/on_Crawl__00_validate_ytdlp.py
Executable file
@@ -0,0 +1,278 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Validation hook for yt-dlp and its dependencies (node, ffmpeg).
|
||||
|
||||
Runs at crawl start to verify yt-dlp and required binaries are available.
|
||||
Outputs JSONL for InstalledBinary and Machine config updates.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import shutil
|
||||
import hashlib
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def get_binary_version(abspath: str, version_flag: str = '--version') -> str | None:
|
||||
"""Get version string from binary."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[abspath, version_flag],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
if result.returncode == 0 and result.stdout:
|
||||
first_line = result.stdout.strip().split('\n')[0]
|
||||
return first_line[:64]
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def get_binary_hash(abspath: str) -> str | None:
|
||||
"""Get SHA256 hash of binary."""
|
||||
try:
|
||||
with open(abspath, 'rb') as f:
|
||||
return hashlib.sha256(f.read()).hexdigest()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def find_ytdlp() -> dict | None:
|
||||
"""Find yt-dlp binary."""
|
||||
try:
|
||||
from abx_pkg import Binary, PipProvider, EnvProvider
|
||||
|
||||
class YtdlpBinary(Binary):
|
||||
name: str = 'yt-dlp'
|
||||
binproviders_supported = [PipProvider(), EnvProvider()]
|
||||
|
||||
binary = YtdlpBinary()
|
||||
loaded = binary.load()
|
||||
if loaded and loaded.abspath:
|
||||
return {
|
||||
'name': 'yt-dlp',
|
||||
'abspath': str(loaded.abspath),
|
||||
'version': str(loaded.version) if loaded.version else None,
|
||||
'sha256': loaded.sha256 if hasattr(loaded, 'sha256') else None,
|
||||
'binprovider': loaded.binprovider.name if loaded.binprovider else 'env',
|
||||
}
|
||||
except ImportError:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback to shutil.which
|
||||
abspath = shutil.which('yt-dlp') or os.environ.get('YTDLP_BINARY', '')
|
||||
if abspath and Path(abspath).is_file():
|
||||
return {
|
||||
'name': 'yt-dlp',
|
||||
'abspath': abspath,
|
||||
'version': get_binary_version(abspath),
|
||||
'sha256': get_binary_hash(abspath),
|
||||
'binprovider': 'env',
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def find_node() -> dict | None:
|
||||
"""Find node binary."""
|
||||
try:
|
||||
from abx_pkg import Binary, AptProvider, BrewProvider, EnvProvider
|
||||
|
||||
class NodeBinary(Binary):
|
||||
name: str = 'node'
|
||||
binproviders_supported = [AptProvider(), BrewProvider(), EnvProvider()]
|
||||
overrides: dict = {'apt': {'packages': ['nodejs']}}
|
||||
|
||||
binary = NodeBinary()
|
||||
loaded = binary.load()
|
||||
if loaded and loaded.abspath:
|
||||
return {
|
||||
'name': 'node',
|
||||
'abspath': str(loaded.abspath),
|
||||
'version': str(loaded.version) if loaded.version else None,
|
||||
'sha256': loaded.sha256 if hasattr(loaded, 'sha256') else None,
|
||||
'binprovider': loaded.binprovider.name if loaded.binprovider else 'env',
|
||||
}
|
||||
except ImportError:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback to shutil.which
|
||||
abspath = shutil.which('node') or os.environ.get('NODE_BINARY', '')
|
||||
if abspath and Path(abspath).is_file():
|
||||
return {
|
||||
'name': 'node',
|
||||
'abspath': abspath,
|
||||
'version': get_binary_version(abspath),
|
||||
'sha256': get_binary_hash(abspath),
|
||||
'binprovider': 'env',
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def find_ffmpeg() -> dict | None:
|
||||
"""Find ffmpeg binary."""
|
||||
try:
|
||||
from abx_pkg import Binary, AptProvider, BrewProvider, EnvProvider
|
||||
|
||||
class FfmpegBinary(Binary):
|
||||
name: str = 'ffmpeg'
|
||||
binproviders_supported = [AptProvider(), BrewProvider(), EnvProvider()]
|
||||
|
||||
binary = FfmpegBinary()
|
||||
loaded = binary.load()
|
||||
if loaded and loaded.abspath:
|
||||
return {
|
||||
'name': 'ffmpeg',
|
||||
'abspath': str(loaded.abspath),
|
||||
'version': str(loaded.version) if loaded.version else None,
|
||||
'sha256': loaded.sha256 if hasattr(loaded, 'sha256') else None,
|
||||
'binprovider': loaded.binprovider.name if loaded.binprovider else 'env',
|
||||
}
|
||||
except ImportError:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback to shutil.which
|
||||
abspath = shutil.which('ffmpeg') or os.environ.get('FFMPEG_BINARY', '')
|
||||
if abspath and Path(abspath).is_file():
|
||||
return {
|
||||
'name': 'ffmpeg',
|
||||
'abspath': abspath,
|
||||
'version': get_binary_version(abspath),
|
||||
'sha256': get_binary_hash(abspath),
|
||||
'binprovider': 'env',
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
# Check for yt-dlp (required)
|
||||
ytdlp_result = find_ytdlp()
|
||||
|
||||
# Check for node (required for JS extraction)
|
||||
node_result = find_node()
|
||||
|
||||
# Check for ffmpeg (required for video conversion)
|
||||
ffmpeg_result = find_ffmpeg()
|
||||
|
||||
missing_deps = []
|
||||
|
||||
# Emit results for yt-dlp
|
||||
if ytdlp_result and ytdlp_result.get('abspath'):
|
||||
print(json.dumps({
|
||||
'type': 'InstalledBinary',
|
||||
'name': ytdlp_result['name'],
|
||||
'abspath': ytdlp_result['abspath'],
|
||||
'version': ytdlp_result['version'],
|
||||
'sha256': ytdlp_result['sha256'],
|
||||
'binprovider': ytdlp_result['binprovider'],
|
||||
}))
|
||||
|
||||
print(json.dumps({
|
||||
'type': 'Machine',
|
||||
'_method': 'update',
|
||||
'key': 'config/YTDLP_BINARY',
|
||||
'value': ytdlp_result['abspath'],
|
||||
}))
|
||||
|
||||
if ytdlp_result['version']:
|
||||
print(json.dumps({
|
||||
'type': 'Machine',
|
||||
'_method': 'update',
|
||||
'key': 'config/YTDLP_VERSION',
|
||||
'value': ytdlp_result['version'],
|
||||
}))
|
||||
else:
|
||||
print(json.dumps({
|
||||
'type': 'Dependency',
|
||||
'bin_name': 'yt-dlp',
|
||||
'bin_providers': 'pip,env',
|
||||
}))
|
||||
missing_deps.append('yt-dlp')
|
||||
|
||||
# Emit results for node
|
||||
if node_result and node_result.get('abspath'):
|
||||
print(json.dumps({
|
||||
'type': 'InstalledBinary',
|
||||
'name': node_result['name'],
|
||||
'abspath': node_result['abspath'],
|
||||
'version': node_result['version'],
|
||||
'sha256': node_result['sha256'],
|
||||
'binprovider': node_result['binprovider'],
|
||||
}))
|
||||
|
||||
print(json.dumps({
|
||||
'type': 'Machine',
|
||||
'_method': 'update',
|
||||
'key': 'config/NODE_BINARY',
|
||||
'value': node_result['abspath'],
|
||||
}))
|
||||
|
||||
if node_result['version']:
|
||||
print(json.dumps({
|
||||
'type': 'Machine',
|
||||
'_method': 'update',
|
||||
'key': 'config/NODE_VERSION',
|
||||
'value': node_result['version'],
|
||||
}))
|
||||
else:
|
||||
print(json.dumps({
|
||||
'type': 'Dependency',
|
||||
'bin_name': 'node',
|
||||
'bin_providers': 'apt,brew,env',
|
||||
}))
|
||||
missing_deps.append('node')
|
||||
|
||||
# Emit results for ffmpeg
|
||||
if ffmpeg_result and ffmpeg_result.get('abspath'):
|
||||
print(json.dumps({
|
||||
'type': 'InstalledBinary',
|
||||
'name': ffmpeg_result['name'],
|
||||
'abspath': ffmpeg_result['abspath'],
|
||||
'version': ffmpeg_result['version'],
|
||||
'sha256': ffmpeg_result['sha256'],
|
||||
'binprovider': ffmpeg_result['binprovider'],
|
||||
}))
|
||||
|
||||
print(json.dumps({
|
||||
'type': 'Machine',
|
||||
'_method': 'update',
|
||||
'key': 'config/FFMPEG_BINARY',
|
||||
'value': ffmpeg_result['abspath'],
|
||||
}))
|
||||
|
||||
if ffmpeg_result['version']:
|
||||
print(json.dumps({
|
||||
'type': 'Machine',
|
||||
'_method': 'update',
|
||||
'key': 'config/FFMPEG_VERSION',
|
||||
'value': ffmpeg_result['version'],
|
||||
}))
|
||||
else:
|
||||
print(json.dumps({
|
||||
'type': 'Dependency',
|
||||
'bin_name': 'ffmpeg',
|
||||
'bin_providers': 'apt,brew,env',
|
||||
}))
|
||||
missing_deps.append('ffmpeg')
|
||||
|
||||
if missing_deps:
|
||||
print(f"Missing dependencies: {', '.join(missing_deps)}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
else:
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
9
archivebox/plugins/media/templates/embed.html
Normal file
9
archivebox/plugins/media/templates/embed.html
Normal file
@@ -0,0 +1,9 @@
|
||||
<!-- Media embed - video/audio player -->
|
||||
<div class="extractor-embed media-embed" style="width: 100%; height: 100%; min-height: 400px; background: #1a1a1a; display: flex; align-items: center; justify-content: center;">
|
||||
<video src="{{ output_path }}"
|
||||
style="max-width: 100%; max-height: 100%;"
|
||||
controls
|
||||
preload="metadata">
|
||||
Your browser does not support the video tag.
|
||||
</video>
|
||||
</div>
|
||||
10
archivebox/plugins/media/templates/fullscreen.html
Normal file
10
archivebox/plugins/media/templates/fullscreen.html
Normal file
@@ -0,0 +1,10 @@
|
||||
<!-- Media fullscreen - full video/audio player -->
|
||||
<div class="extractor-fullscreen media-fullscreen" style="width: 100%; height: 100vh; background: #000; display: flex; align-items: center; justify-content: center;">
|
||||
<video src="{{ output_path }}"
|
||||
style="max-width: 100%; max-height: 100%;"
|
||||
controls
|
||||
autoplay
|
||||
preload="auto">
|
||||
Your browser does not support the video tag.
|
||||
</video>
|
||||
</div>
|
||||
1
archivebox/plugins/media/templates/icon.html
Normal file
1
archivebox/plugins/media/templates/icon.html
Normal file
@@ -0,0 +1 @@
|
||||
🎬
|
||||
14
archivebox/plugins/media/templates/thumbnail.html
Normal file
14
archivebox/plugins/media/templates/thumbnail.html
Normal file
@@ -0,0 +1,14 @@
|
||||
<!-- Media thumbnail - shows video/audio player or placeholder -->
|
||||
<div class="extractor-thumbnail media-thumbnail" style="width: 100%; height: 100px; overflow: hidden; background: #1a1a1a; display: flex; align-items: center; justify-content: center;">
|
||||
<video src="{{ output_path }}"
|
||||
style="width: 100%; height: 100px; object-fit: contain;"
|
||||
poster=""
|
||||
preload="metadata"
|
||||
muted
|
||||
onerror="this.style.display='none'; this.nextElementSibling.style.display='flex';">
|
||||
</video>
|
||||
<div style="display: none; flex-direction: column; align-items: center; color: #888; font-size: 12px;">
|
||||
<span style="font-size: 32px;">🎬</span>
|
||||
<span>Media</span>
|
||||
</div>
|
||||
</div>
|
||||
@@ -21,7 +21,7 @@ import pytest
|
||||
PLUGIN_DIR = Path(__file__).parent.parent
|
||||
PLUGINS_ROOT = PLUGIN_DIR.parent
|
||||
MEDIA_HOOK = PLUGIN_DIR / 'on_Snapshot__51_media.py'
|
||||
MEDIA_INSTALL_HOOK = PLUGIN_DIR / 'on_Crawl__00_install_ytdlp.py'
|
||||
MEDIA_VALIDATE_HOOK = PLUGIN_DIR / 'on_Crawl__00_validate_ytdlp.py'
|
||||
TEST_URL = 'https://example.com/video.mp4'
|
||||
|
||||
def test_hook_script_exists():
|
||||
@@ -29,46 +29,72 @@ def test_hook_script_exists():
|
||||
assert MEDIA_HOOK.exists(), f"Hook not found: {MEDIA_HOOK}"
|
||||
|
||||
|
||||
def test_ytdlp_install_hook():
|
||||
"""Test yt-dlp install hook to install yt-dlp if needed."""
|
||||
# Run yt-dlp install hook
|
||||
def test_ytdlp_validate_hook():
|
||||
"""Test yt-dlp validate hook checks for yt-dlp and dependencies (node, ffmpeg)."""
|
||||
# Run yt-dlp validate hook
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(MEDIA_INSTALL_HOOK)],
|
||||
[sys.executable, str(MEDIA_VALIDATE_HOOK)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=600
|
||||
timeout=30
|
||||
)
|
||||
|
||||
assert result.returncode == 0, f"Install hook failed: {result.stderr}"
|
||||
# Hook exits 0 if all binaries found, 1 if any not found
|
||||
# Parse output for InstalledBinary and Dependency records
|
||||
found_binaries = {'node': False, 'ffmpeg': False, 'yt-dlp': False}
|
||||
found_dependencies = {'node': False, 'ffmpeg': False, 'yt-dlp': False}
|
||||
|
||||
# Verify InstalledBinary JSONL output
|
||||
found_binary = False
|
||||
for line in result.stdout.strip().split('\n'):
|
||||
if line.strip():
|
||||
try:
|
||||
record = json.loads(line)
|
||||
if record.get('type') == 'InstalledBinary':
|
||||
assert record['name'] == 'yt-dlp'
|
||||
assert record['abspath']
|
||||
found_binary = True
|
||||
break
|
||||
name = record['name']
|
||||
if name in found_binaries:
|
||||
assert record['abspath'], f"{name} should have abspath"
|
||||
found_binaries[name] = True
|
||||
elif record.get('type') == 'Dependency':
|
||||
name = record['bin_name']
|
||||
if name in found_dependencies:
|
||||
found_dependencies[name] = True
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
assert found_binary, "Should output InstalledBinary record"
|
||||
# Each binary should either be found (InstalledBinary) or missing (Dependency)
|
||||
for binary_name in ['yt-dlp', 'node', 'ffmpeg']:
|
||||
assert found_binaries[binary_name] or found_dependencies[binary_name], \
|
||||
f"{binary_name} should have either InstalledBinary or Dependency record"
|
||||
|
||||
|
||||
def test_verify_deps_with_abx_pkg():
|
||||
"""Verify yt-dlp is available via abx-pkg after hook installation."""
|
||||
from abx_pkg import Binary, PipProvider, EnvProvider, BinProviderOverrides
|
||||
"""Verify yt-dlp, node, and ffmpeg are available via abx-pkg."""
|
||||
from abx_pkg import Binary, PipProvider, AptProvider, BrewProvider, EnvProvider, BinProviderOverrides
|
||||
|
||||
PipProvider.model_rebuild()
|
||||
EnvProvider.model_rebuild()
|
||||
missing_binaries = []
|
||||
|
||||
# Verify yt-dlp is available
|
||||
ytdlp_binary = Binary(name='yt-dlp', binproviders=[PipProvider(), EnvProvider()])
|
||||
ytdlp_loaded = ytdlp_binary.load()
|
||||
assert ytdlp_loaded and ytdlp_loaded.abspath, "yt-dlp should be available after install hook"
|
||||
if not (ytdlp_loaded and ytdlp_loaded.abspath):
|
||||
missing_binaries.append('yt-dlp')
|
||||
|
||||
# Verify node is available (yt-dlp needs it for JS extraction)
|
||||
node_binary = Binary(
|
||||
name='node',
|
||||
binproviders=[AptProvider(), BrewProvider(), EnvProvider()]
|
||||
)
|
||||
node_loaded = node_binary.load()
|
||||
if not (node_loaded and node_loaded.abspath):
|
||||
missing_binaries.append('node')
|
||||
|
||||
# Verify ffmpeg is available (yt-dlp needs it for video conversion)
|
||||
ffmpeg_binary = Binary(name='ffmpeg', binproviders=[AptProvider(), BrewProvider(), EnvProvider()])
|
||||
ffmpeg_loaded = ffmpeg_binary.load()
|
||||
if not (ffmpeg_loaded and ffmpeg_loaded.abspath):
|
||||
missing_binaries.append('ffmpeg')
|
||||
|
||||
if missing_binaries:
|
||||
pytest.skip(f"Binaries not available: {', '.join(missing_binaries)} - Dependency records should have been emitted")
|
||||
|
||||
def test_handles_non_media_url():
|
||||
"""Test that media extractor handles non-media URLs gracefully via hook."""
|
||||
|
||||
Reference in New Issue
Block a user