rename media plugin to ytdlp with backwards-compatible aliases

- Rename archivebox/plugins/media/ → archivebox/plugins/ytdlp/
- Rename hook script on_Snapshot__63_media.bg.py → on_Snapshot__63_ytdlp.bg.py
- Update config.json: YTDLP_* as primary keys, MEDIA_* as x-aliases
- Update templates CSS classes: media-* → ytdlp-*
- Fix gallerydl bug: remove incorrect dependency on media plugin output
- Update all codebase references to use YTDLP_* and SAVE_YTDLP
- Add backwards compatibility test for MEDIA_ENABLED alias

archivebox/plugins/ytdlp/binaries.jsonl

@@ -0,0 +1,3 @@
+{"type": "Binary", "name": "yt-dlp", "binproviders": "pip,brew,apt,env"}
+{"type": "Binary", "name": "node", "binproviders": "apt,brew,env", "overrides": {"apt": {"packages": ["nodejs"]}}}
+{"type": "Binary", "name": "ffmpeg", "binproviders": "apt,brew,env"}

archivebox/plugins/ytdlp/config.json

@@ -0,0 +1,60 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "additionalProperties": false,
+  "properties": {
+    "YTDLP_ENABLED": {
+      "type": "boolean",
+      "default": true,
+      "x-aliases": ["MEDIA_ENABLED", "SAVE_MEDIA", "USE_MEDIA", "USE_YTDLP", "FETCH_MEDIA", "SAVE_YTDLP"],
+      "description": "Enable video/audio downloading with yt-dlp"
+    },
+    "YTDLP_BINARY": {
+      "type": "string",
+      "default": "yt-dlp",
+      "x-aliases": ["MEDIA_BINARY", "YOUTUBEDL_BINARY", "YOUTUBE_DL_BINARY"],
+      "description": "Path to yt-dlp binary"
+    },
+    "YTDLP_TIMEOUT": {
+      "type": "integer",
+      "default": 3600,
+      "minimum": 30,
+      "x-fallback": "TIMEOUT",
+      "x-aliases": ["MEDIA_TIMEOUT"],
+      "description": "Timeout for yt-dlp downloads in seconds"
+    },
+    "YTDLP_MAX_SIZE": {
+      "type": "string",
+      "default": "750m",
+      "pattern": "^\\d+[kmgKMG]?$",
+      "x-aliases": ["MEDIA_MAX_SIZE"],
+      "description": "Maximum file size for yt-dlp downloads"
+    },
+    "YTDLP_CHECK_SSL_VALIDITY": {
+      "type": "boolean",
+      "default": true,
+      "x-fallback": "CHECK_SSL_VALIDITY",
+      "x-aliases": ["MEDIA_CHECK_SSL_VALIDITY"],
+      "description": "Whether to verify SSL certificates"
+    },
+    "YTDLP_ARGS": {
+      "type": "array",
+      "items": {"type": "string"},
+      "default": [
+        "--write-info-json",
+        "--write-thumbnail",
+        "--write-sub",
+        "--embed-subs",
+        "--write-auto-sub"
+      ],
+      "x-aliases": ["MEDIA_ARGS"],
+      "description": "Default yt-dlp arguments"
+    },
+    "YTDLP_EXTRA_ARGS": {
+      "type": "string",
+      "default": "",
+      "x-aliases": ["MEDIA_EXTRA_ARGS"],
+      "description": "Extra arguments for yt-dlp (space-separated)"
+    }
+  }
+}

archivebox/plugins/ytdlp/on_Snapshot__63_ytdlp.bg.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+"""
+Download video/audio from a URL using yt-dlp.
+
+Usage: on_Snapshot__ytdlp.py --url=<url> --snapshot-id=<uuid>
+Output: Downloads video/audio files to $PWD/ytdlp/
+
+Environment variables:
+    YTDLP_BINARY: Path to yt-dlp binary
+    YTDLP_TIMEOUT: Timeout in seconds (default: 3600 for large downloads)
+    YTDLP_CHECK_SSL_VALIDITY: Whether to check SSL certificates (default: True)
+    YTDLP_EXTRA_ARGS: Extra arguments for yt-dlp (space-separated)
+    YTDLP_MAX_SIZE: Maximum file size (default: 750m)
+
+    # Feature toggles (with backwards-compatible aliases)
+    YTDLP_ENABLED: Enable yt-dlp extraction (default: True)
+    SAVE_YTDLP: Alias for YTDLP_ENABLED
+    MEDIA_ENABLED: Backwards-compatible alias for YTDLP_ENABLED
+
+    # Fallback to ARCHIVING_CONFIG values if YTDLP_* not set:
+    TIMEOUT: Fallback timeout
+    CHECK_SSL_VALIDITY: Fallback SSL check
+"""
+
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import rich_click as click
+
+
+# Extractor metadata
+PLUGIN_NAME = 'ytdlp'
+BIN_NAME = 'yt-dlp'
+BIN_PROVIDERS = 'pip,apt,brew,env'
+OUTPUT_DIR = '.'
+
+
+def get_env(name: str, default: str = '') -> str:
+    return os.environ.get(name, default).strip()
+
+
+def get_env_bool(name: str, default: bool = False) -> bool:
+    val = get_env(name, '').lower()
+    if val in ('true', '1', 'yes', 'on'):
+        return True
+    if val in ('false', '0', 'no', 'off'):
+        return False
+    return default
+
+
+def get_env_int(name: str, default: int = 0) -> int:
+    try:
+        return int(get_env(name, str(default)))
+    except ValueError:
+        return default
+
+
+STATICFILE_DIR = '../staticfile'
+
+def has_staticfile_output() -> bool:
+    """Check if staticfile extractor already downloaded this URL."""
+    staticfile_dir = Path(STATICFILE_DIR)
+    return staticfile_dir.exists() and any(staticfile_dir.iterdir())
+
+
+# Default yt-dlp args (from old YTDLP_CONFIG)
+def get_ytdlp_default_args(max_size: str = '750m') -> list[str]:
+    """Build default yt-dlp arguments."""
+    return [
+        '--restrict-filenames',
+        '--trim-filenames', '128',
+        '--write-description',
+        '--write-info-json',
+        '--write-thumbnail',
+        '--write-sub',
+        '--write-auto-subs',
+        '--convert-subs=srt',
+        '--yes-playlist',
+        '--continue',
+        '--no-abort-on-error',
+        '--ignore-errors',
+        '--geo-bypass',
+        '--add-metadata',
+        f'--format=(bv*+ba/b)[filesize<={max_size}][filesize_approx<=?{max_size}]/(bv*+ba/b)',
+    ]
+
+
+def save_ytdlp(url: str, binary: str) -> tuple[bool, str | None, str]:
+    """
+    Download video/audio using yt-dlp.
+
+    Returns: (success, output_path, error_message)
+    """
+    # Get config from env (YTDLP_* primary, MEDIA_* as fallback via aliases)
+    timeout = get_env_int('TIMEOUT', 3600)
+    check_ssl = get_env_bool('CHECK_SSL_VALIDITY', True)
+    extra_args = get_env('YTDLP_EXTRA_ARGS', '')
+    max_size = get_env('YTDLP_MAX_SIZE', '') or get_env('MEDIA_MAX_SIZE', '750m')
+
+    # Output directory is current directory (hook already runs in output dir)
+    output_dir = Path(OUTPUT_DIR)
+
+    # Build command (later options take precedence)
+    cmd = [
+        binary,
+        *get_ytdlp_default_args(max_size),
+        '--no-progress',
+        '-o', '%(title)s.%(ext)s',
+    ]
+
+    if not check_ssl:
+        cmd.append('--no-check-certificate')
+
+    if extra_args:
+        cmd.extend(extra_args.split())
+
+    cmd.append(url)
+
+    try:
+        result = subprocess.run(cmd, capture_output=True, timeout=timeout, text=True)
+
+        # Check if any media files were downloaded
+        media_extensions = (
+            '.mp4', '.webm', '.mkv', '.avi', '.mov', '.flv', '.wmv', '.m4v',
+            '.mp3', '.m4a', '.ogg', '.wav', '.flac', '.aac', '.opus',
+            '.json', '.jpg', '.png', '.webp', '.jpeg',
+            '.vtt', '.srt', '.ass', '.lrc',
+            '.description',
+        )
+
+        downloaded_files = [
+            f for f in output_dir.glob('*')
+            if f.is_file() and f.suffix.lower() in media_extensions
+        ]
+
+        if downloaded_files:
+            # Return first video/audio file, or first file if no media
+            video_audio = [
+                f for f in downloaded_files
+                if f.suffix.lower() in ('.mp4', '.webm', '.mkv', '.avi', '.mov', '.mp3', '.m4a', '.ogg', '.wav', '.flac')
+            ]
+            output = str(video_audio[0]) if video_audio else str(downloaded_files[0])
+            return True, output, ''
+        else:
+            stderr = result.stderr
+
+            # These are NOT errors - page simply has no downloadable media
+            # Return success with no output (legitimate "nothing to download")
+            if 'ERROR: Unsupported URL' in stderr:
+                return True, None, ''  # Not a media site - success, no output
+            if 'URL could be a direct video link' in stderr:
+                return True, None, ''  # Not a supported media URL - success, no output
+            if result.returncode == 0:
+                return True, None, ''  # yt-dlp exited cleanly, just no media - success
+
+            # These ARE errors - something went wrong
+            if 'HTTP Error 404' in stderr:
+                return False, None, '404 Not Found'
+            if 'HTTP Error 403' in stderr:
+                return False, None, '403 Forbidden'
+            if 'Unable to extract' in stderr:
+                return False, None, 'Unable to extract media info'
+
+            return False, None, f'yt-dlp error: {stderr[:200]}'
+
+    except subprocess.TimeoutExpired:
+        return False, None, f'Timed out after {timeout} seconds'
+    except Exception as e:
+        return False, None, f'{type(e).__name__}: {e}'
+
+
+@click.command()
+@click.option('--url', required=True, help='URL to download video/audio from')
+@click.option('--snapshot-id', required=True, help='Snapshot UUID')
+def main(url: str, snapshot_id: str):
+    """Download video/audio from a URL using yt-dlp."""
+
+    try:
+        # Check if yt-dlp downloading is enabled (YTDLP_ENABLED primary, MEDIA_ENABLED fallback)
+        ytdlp_enabled = get_env_bool('YTDLP_ENABLED', True) and get_env_bool('MEDIA_ENABLED', True)
+        if not ytdlp_enabled:
+            print('Skipping ytdlp (YTDLP_ENABLED=False)', file=sys.stderr)
+            # Temporary failure (config disabled) - NO JSONL emission
+            sys.exit(0)
+
+        # Check if staticfile extractor already handled this (permanent skip)
+        if has_staticfile_output():
+            print('Skipping ytdlp - staticfile extractor already downloaded this', file=sys.stderr)
+            print(json.dumps({'type': 'ArchiveResult', 'status': 'skipped', 'output_str': 'staticfile already exists'}))
+            sys.exit(0)
+
+        # Get binary from environment
+        binary = get_env('YTDLP_BINARY', 'yt-dlp')
+
+        # Run extraction
+        success, output, error = save_ytdlp(url, binary)
+
+        if success:
+            # Success - emit ArchiveResult
+            result = {
+                'type': 'ArchiveResult',
+                'status': 'succeeded',
+                'output_str': output or ''
+            }
+            print(json.dumps(result))
+            sys.exit(0)
+        else:
+            # Transient error - emit NO JSONL
+            print(f'ERROR: {error}', file=sys.stderr)
+            sys.exit(1)
+
+    except Exception as e:
+        # Transient error - emit NO JSONL
+        print(f'ERROR: {type(e).__name__}: {e}', file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

archivebox/plugins/ytdlp/templates/embed.html

@@ -0,0 +1,9 @@
+<!-- YT-DLP embed - video/audio player -->
+<div class="extractor-embed ytdlp-embed" style="width: 100%; height: 100%; min-height: 400px; background: #1a1a1a; display: flex; align-items: center; justify-content: center;">
+    <video src="{{ output_path }}"
+           style="max-width: 100%; max-height: 100%;"
+           controls
+           preload="metadata">
+        Your browser does not support the video tag.
+    </video>
+</div>

archivebox/plugins/ytdlp/templates/fullscreen.html

@@ -0,0 +1,10 @@
+<!-- YT-DLP fullscreen - full video/audio player -->
+<div class="extractor-fullscreen ytdlp-fullscreen" style="width: 100%; height: 100vh; background: #000; display: flex; align-items: center; justify-content: center;">
+    <video src="{{ output_path }}"
+           style="max-width: 100%; max-height: 100%;"
+           controls
+           autoplay
+           preload="auto">
+        Your browser does not support the video tag.
+    </video>
+</div>

archivebox/plugins/ytdlp/templates/icon.html

@@ -0,0 +1 @@
+🎬

archivebox/plugins/ytdlp/templates/thumbnail.html

@@ -0,0 +1,14 @@
+<!-- YT-DLP thumbnail - shows video/audio player or placeholder -->
+<div class="extractor-thumbnail ytdlp-thumbnail" style="width: 100%; height: 100px; overflow: hidden; background: #1a1a1a; display: flex; align-items: center; justify-content: center;">
+    <video src="{{ output_path }}"
+           style="width: 100%; height: 100px; object-fit: contain;"
+           poster=""
+           preload="metadata"
+           muted
+           onerror="this.style.display='none'; this.nextElementSibling.style.display='flex';">
+    </video>
+    <div style="display: none; flex-direction: column; align-items: center; color: #888; font-size: 12px;">
+        <span style="font-size: 32px;">🎬</span>
+        <span>YT-DLP</span>
+    </div>
+</div>

archivebox/plugins/ytdlp/tests/test_ytdlp.py

@@ -0,0 +1,227 @@
+"""
+Integration tests for ytdlp plugin
+
+Tests verify:
+1. Hook script exists
+2. Dependencies installed via validation hooks
+3. Verify deps with abx-pkg
+4. YT-DLP extraction works on video URLs
+5. JSONL output is correct
+6. Config options work (YTDLP_* and backwards-compatible MEDIA_* aliases)
+7. Handles non-video URLs gracefully
+"""
+
+import json
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+import pytest
+
+PLUGIN_DIR = Path(__file__).parent.parent
+PLUGINS_ROOT = PLUGIN_DIR.parent
+YTDLP_HOOK = next(PLUGIN_DIR.glob('on_Snapshot__*_ytdlp.*'), None)
+TEST_URL = 'https://example.com/video.mp4'
+
+def test_hook_script_exists():
+    """Verify on_Snapshot hook exists."""
+    assert YTDLP_HOOK.exists(), f"Hook not found: {YTDLP_HOOK}"
+
+
+def test_verify_deps_with_abx_pkg():
+    """Verify yt-dlp, node, and ffmpeg are available via abx-pkg."""
+    from abx_pkg import Binary, PipProvider, AptProvider, BrewProvider, EnvProvider, BinProviderOverrides
+
+    missing_binaries = []
+
+    # Verify yt-dlp is available
+    ytdlp_binary = Binary(name='yt-dlp', binproviders=[PipProvider(), EnvProvider()])
+    ytdlp_loaded = ytdlp_binary.load()
+    if not (ytdlp_loaded and ytdlp_loaded.abspath):
+        missing_binaries.append('yt-dlp')
+
+    # Verify node is available (yt-dlp needs it for JS extraction)
+    node_binary = Binary(
+        name='node',
+        binproviders=[AptProvider(), BrewProvider(), EnvProvider()]
+    )
+    node_loaded = node_binary.load()
+    if not (node_loaded and node_loaded.abspath):
+        missing_binaries.append('node')
+
+    # Verify ffmpeg is available (yt-dlp needs it for video conversion)
+    ffmpeg_binary = Binary(name='ffmpeg', binproviders=[AptProvider(), BrewProvider(), EnvProvider()])
+    ffmpeg_loaded = ffmpeg_binary.load()
+    if not (ffmpeg_loaded and ffmpeg_loaded.abspath):
+        missing_binaries.append('ffmpeg')
+
+    if missing_binaries:
+        pass
+
+def test_handles_non_video_url():
+    """Test that ytdlp extractor handles non-video URLs gracefully via hook."""
+    # Prerequisites checked by earlier test
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tmpdir = Path(tmpdir)
+
+        # Run ytdlp extraction hook on non-video URL
+        result = subprocess.run(
+            [sys.executable, str(YTDLP_HOOK), '--url', 'https://example.com', '--snapshot-id', 'test789'],
+            cwd=tmpdir,
+            capture_output=True,
+            text=True,
+            timeout=60
+        )
+
+        # Should exit 0 even for non-media URL
+        assert result.returncode == 0, f"Should handle non-media URL gracefully: {result.stderr}"
+
+        # Parse clean JSONL output
+        result_json = None
+        for line in result.stdout.strip().split('\n'):
+            line = line.strip()
+            if line.startswith('{'):
+                pass
+                try:
+                    record = json.loads(line)
+                    if record.get('type') == 'ArchiveResult':
+                        result_json = record
+                        break
+                except json.JSONDecodeError:
+                    pass
+
+        assert result_json, "Should have ArchiveResult JSONL output"
+        assert result_json['status'] == 'succeeded', f"Should succeed: {result_json}"
+
+
+def test_config_ytdlp_enabled_false_skips():
+    """Test that YTDLP_ENABLED=False exits without emitting JSONL."""
+    import os
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        env = os.environ.copy()
+        env['YTDLP_ENABLED'] = 'False'
+
+        result = subprocess.run(
+            [sys.executable, str(YTDLP_HOOK), '--url', TEST_URL, '--snapshot-id', 'test999'],
+            cwd=tmpdir,
+            capture_output=True,
+            text=True,
+            env=env,
+            timeout=30
+        )
+
+        assert result.returncode == 0, f"Should exit 0 when feature disabled: {result.stderr}"
+
+        # Feature disabled - temporary failure, should NOT emit JSONL
+        assert 'Skipping' in result.stderr or 'False' in result.stderr, "Should log skip reason to stderr"
+
+        # Should NOT emit any JSONL
+        jsonl_lines = [line for line in result.stdout.strip().split('\n') if line.strip().startswith('{')]
+        assert len(jsonl_lines) == 0, f"Should not emit JSONL when feature disabled, but got: {jsonl_lines}"
+
+
+def test_config_media_enabled_backwards_compat():
+    """Test that MEDIA_ENABLED=False (backwards-compatible alias) also works."""
+    import os
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        env = os.environ.copy()
+        env['MEDIA_ENABLED'] = 'False'
+
+        result = subprocess.run(
+            [sys.executable, str(YTDLP_HOOK), '--url', TEST_URL, '--snapshot-id', 'test_compat'],
+            cwd=tmpdir,
+            capture_output=True,
+            text=True,
+            env=env,
+            timeout=30
+        )
+
+        assert result.returncode == 0, f"Should exit 0 when feature disabled via MEDIA_ENABLED: {result.stderr}"
+
+        # Should NOT emit any JSONL when disabled via backwards-compatible alias
+        jsonl_lines = [line for line in result.stdout.strip().split('\n') if line.strip().startswith('{')]
+        assert len(jsonl_lines) == 0, f"Should not emit JSONL when feature disabled via MEDIA_ENABLED, but got: {jsonl_lines}"
+
+
+def test_config_timeout():
+    """Test that YTDLP_TIMEOUT config is respected (also via MEDIA_TIMEOUT alias)."""
+    import os
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        env = os.environ.copy()
+        env['YTDLP_TIMEOUT'] = '5'
+
+        start_time = time.time()
+        result = subprocess.run(
+            [sys.executable, str(YTDLP_HOOK), '--url', 'https://example.com', '--snapshot-id', 'testtimeout'],
+            cwd=tmpdir,
+            capture_output=True,
+            text=True,
+            env=env,
+            timeout=10  # Should complete in 5s, use 10s as safety margin
+        )
+        elapsed_time = time.time() - start_time
+
+        assert result.returncode == 0, f"Should complete without hanging: {result.stderr}"
+        # Allow 1 second overhead for subprocess startup and Python interpreter
+        assert elapsed_time <= 6.0, f"Should complete within 6 seconds (5s timeout + 1s overhead), took {elapsed_time:.2f}s"
+
+
+def test_real_youtube_url():
+    """Test that yt-dlp can extract video/audio from a real YouTube URL."""
+    import os
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tmpdir = Path(tmpdir)
+
+        # Use a short, stable YouTube video (YouTube's own about video)
+        youtube_url = 'https://www.youtube.com/watch?v=jNQXAC9IVRw'  # "Me at the zoo" - first YouTube video
+
+        env = os.environ.copy()
+        env['YTDLP_TIMEOUT'] = '120'  # Give it time to download
+
+        start_time = time.time()
+        result = subprocess.run(
+            [sys.executable, str(YTDLP_HOOK), '--url', youtube_url, '--snapshot-id', 'testyoutube'],
+            cwd=tmpdir,
+            capture_output=True,
+            text=True,
+            env=env,
+            timeout=180
+        )
+        elapsed_time = time.time() - start_time
+
+        # Should succeed
+        assert result.returncode == 0, f"Should extract video/audio successfully: {result.stderr}"
+
+        # Parse JSONL output
+        result_json = None
+        for line in result.stdout.strip().split('\n'):
+            line = line.strip()
+            if line.startswith('{'):
+                try:
+                    record = json.loads(line)
+                    if record.get('type') == 'ArchiveResult':
+                        result_json = record
+                        break
+                except json.JSONDecodeError:
+                    pass
+
+        assert result_json, f"Should have ArchiveResult JSONL output. stdout: {result.stdout}"
+        assert result_json['status'] == 'succeeded', f"Should succeed: {result_json}"
+
+        # Check that some video/audio files were downloaded
+        output_files = list(tmpdir.glob('**/*'))
+        media_files = [f for f in output_files if f.is_file() and f.suffix.lower() in ('.mp4', '.webm', '.mkv', '.m4a', '.mp3', '.json', '.jpg', '.webp')]
+
+        assert len(media_files) > 0, f"Should have downloaded at least one video/audio file. Files: {output_files}"
+
+        print(f"Successfully extracted {len(media_files)} file(s) in {elapsed_time:.2f}s")
+
+
+if __name__ == '__main__':
+    pytest.main([__file__, '-v'])