alex/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-04-06 07:47:53 +10:00
Code Issues Packages Projects Releases Wiki Activity

rename media plugin to ytdlp with backwards-compatible aliases

Browse Source 
- Rename archivebox/plugins/media/ → archivebox/plugins/ytdlp/
- Rename hook script on_Snapshot__63_media.bg.py → on_Snapshot__63_ytdlp.bg.py
- Update config.json: YTDLP_* as primary keys, MEDIA_* as x-aliases
- Update templates CSS classes: media-* → ytdlp-*
- Fix gallerydl bug: remove incorrect dependency on media plugin output
- Update all codebase references to use YTDLP_* and SAVE_YTDLP
- Add backwards compatibility test for MEDIA_ENABLED alias
This commit is contained in:
Claude
2025-12-29 19:09:05 +00:00
parent 30c60eef76
commit a5654e877f
19 changed files with 110 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

227
archivebox/plugins/ytdlp/tests/test_ytdlp.py Normal file
View File

@@ -0,0 +1,227 @@
"""
Integration tests for ytdlp plugin
Tests verify:
1. Hook script exists
2. Dependencies installed via validation hooks
3. Verify deps with abx-pkg
4. YT-DLP extraction works on video URLs
5. JSONL output is correct
6. Config options work (YTDLP_* and backwards-compatible MEDIA_* aliases)
7. Handles non-video URLs gracefully
"""
import json
import subprocess
import sys
import tempfile
import time
from pathlib import Path
import pytest
PLUGIN_DIR = Path(__file__).parent.parent
PLUGINS_ROOT = PLUGIN_DIR.parent
YTDLP_HOOK = next(PLUGIN_DIR.glob('on_Snapshot__*_ytdlp.*'), None)
TEST_URL = 'https://example.com/video.mp4'
def test_hook_script_exists():
"""Verify on_Snapshot hook exists."""
assert YTDLP_HOOK.exists(), f"Hook not found: {YTDLP_HOOK}"
def test_verify_deps_with_abx_pkg():
"""Verify yt-dlp, node, and ffmpeg are available via abx-pkg."""
from abx_pkg import Binary, PipProvider, AptProvider, BrewProvider, EnvProvider, BinProviderOverrides
missing_binaries = []
# Verify yt-dlp is available
ytdlp_binary = Binary(name='yt-dlp', binproviders=[PipProvider(), EnvProvider()])
ytdlp_loaded = ytdlp_binary.load()
if not (ytdlp_loaded and ytdlp_loaded.abspath):
missing_binaries.append('yt-dlp')
# Verify node is available (yt-dlp needs it for JS extraction)
node_binary = Binary(
name='node',
binproviders=[AptProvider(), BrewProvider(), EnvProvider()]
)
node_loaded = node_binary.load()
if not (node_loaded and node_loaded.abspath):
missing_binaries.append('node')
# Verify ffmpeg is available (yt-dlp needs it for video conversion)
ffmpeg_binary = Binary(name='ffmpeg', binproviders=[AptProvider(), BrewProvider(), EnvProvider()])
ffmpeg_loaded = ffmpeg_binary.load()
if not (ffmpeg_loaded and ffmpeg_loaded.abspath):
missing_binaries.append('ffmpeg')
if missing_binaries:
pass
def test_handles_non_video_url():
"""Test that ytdlp extractor handles non-video URLs gracefully via hook."""
# Prerequisites checked by earlier test
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir = Path(tmpdir)
# Run ytdlp extraction hook on non-video URL
result = subprocess.run(
[sys.executable, str(YTDLP_HOOK), '--url', 'https://example.com', '--snapshot-id', 'test789'],
cwd=tmpdir,
capture_output=True,
text=True,
timeout=60
)
# Should exit 0 even for non-media URL
assert result.returncode == 0, f"Should handle non-media URL gracefully: {result.stderr}"
# Parse clean JSONL output
result_json = None
for line in result.stdout.strip().split('\n'):
line = line.strip()
if line.startswith('{'):
pass
try:
record = json.loads(line)
if record.get('type') == 'ArchiveResult':
result_json = record
break
except json.JSONDecodeError:
pass
assert result_json, "Should have ArchiveResult JSONL output"
assert result_json['status'] == 'succeeded', f"Should succeed: {result_json}"
def test_config_ytdlp_enabled_false_skips():
"""Test that YTDLP_ENABLED=False exits without emitting JSONL."""
import os
with tempfile.TemporaryDirectory() as tmpdir:
env = os.environ.copy()
env['YTDLP_ENABLED'] = 'False'
result = subprocess.run(
[sys.executable, str(YTDLP_HOOK), '--url', TEST_URL, '--snapshot-id', 'test999'],
cwd=tmpdir,
capture_output=True,
text=True,
env=env,
timeout=30
)
assert result.returncode == 0, f"Should exit 0 when feature disabled: {result.stderr}"
# Feature disabled - temporary failure, should NOT emit JSONL
assert 'Skipping' in result.stderr or 'False' in result.stderr, "Should log skip reason to stderr"
# Should NOT emit any JSONL
jsonl_lines = [line for line in result.stdout.strip().split('\n') if line.strip().startswith('{')]
assert len(jsonl_lines) == 0, f"Should not emit JSONL when feature disabled, but got: {jsonl_lines}"
def test_config_media_enabled_backwards_compat():
"""Test that MEDIA_ENABLED=False (backwards-compatible alias) also works."""
import os
with tempfile.TemporaryDirectory() as tmpdir:
env = os.environ.copy()
env['MEDIA_ENABLED'] = 'False'
result = subprocess.run(
[sys.executable, str(YTDLP_HOOK), '--url', TEST_URL, '--snapshot-id', 'test_compat'],
cwd=tmpdir,
capture_output=True,
text=True,
env=env,
timeout=30
)
assert result.returncode == 0, f"Should exit 0 when feature disabled via MEDIA_ENABLED: {result.stderr}"
# Should NOT emit any JSONL when disabled via backwards-compatible alias
jsonl_lines = [line for line in result.stdout.strip().split('\n') if line.strip().startswith('{')]
assert len(jsonl_lines) == 0, f"Should not emit JSONL when feature disabled via MEDIA_ENABLED, but got: {jsonl_lines}"
def test_config_timeout():
"""Test that YTDLP_TIMEOUT config is respected (also via MEDIA_TIMEOUT alias)."""
import os
with tempfile.TemporaryDirectory() as tmpdir:
env = os.environ.copy()
env['YTDLP_TIMEOUT'] = '5'
start_time = time.time()
result = subprocess.run(
[sys.executable, str(YTDLP_HOOK), '--url', 'https://example.com', '--snapshot-id', 'testtimeout'],
cwd=tmpdir,
capture_output=True,
text=True,
env=env,
timeout=10 # Should complete in 5s, use 10s as safety margin
)
elapsed_time = time.time() - start_time
assert result.returncode == 0, f"Should complete without hanging: {result.stderr}"
# Allow 1 second overhead for subprocess startup and Python interpreter
assert elapsed_time <= 6.0, f"Should complete within 6 seconds (5s timeout + 1s overhead), took {elapsed_time:.2f}s"
def test_real_youtube_url():
"""Test that yt-dlp can extract video/audio from a real YouTube URL."""
import os
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir = Path(tmpdir)
# Use a short, stable YouTube video (YouTube's own about video)
youtube_url = 'https://www.youtube.com/watch?v=jNQXAC9IVRw' # "Me at the zoo" - first YouTube video
env = os.environ.copy()
env['YTDLP_TIMEOUT'] = '120' # Give it time to download
start_time = time.time()
result = subprocess.run(
[sys.executable, str(YTDLP_HOOK), '--url', youtube_url, '--snapshot-id', 'testyoutube'],
cwd=tmpdir,
capture_output=True,
text=True,
env=env,
timeout=180
)
elapsed_time = time.time() - start_time
# Should succeed
assert result.returncode == 0, f"Should extract video/audio successfully: {result.stderr}"
# Parse JSONL output
result_json = None
for line in result.stdout.strip().split('\n'):
line = line.strip()
if line.startswith('{'):
try:
record = json.loads(line)
if record.get('type') == 'ArchiveResult':
result_json = record
break
except json.JSONDecodeError:
pass
assert result_json, f"Should have ArchiveResult JSONL output. stdout: {result.stdout}"
assert result_json['status'] == 'succeeded', f"Should succeed: {result_json}"
# Check that some video/audio files were downloaded
output_files = list(tmpdir.glob('**/*'))
media_files = [f for f in output_files if f.is_file() and f.suffix.lower() in ('.mp4', '.webm', '.mkv', '.m4a', '.mp3', '.json', '.jpg', '.webp')]
assert len(media_files) > 0, f"Should have downloaded at least one video/audio file. Files: {output_files}"
print(f"Successfully extracted {len(media_files)} file(s) in {elapsed_time:.2f}s")
if __name__ == '__main__':
pytest.main([__file__, '-v'])