rename media plugin to ytdlp with backwards-compatible aliases

- Rename archivebox/plugins/media/ → archivebox/plugins/ytdlp/ - Rename hook script on_Snapshot__63_media.bg.py → on_Snapshot__63_ytdlp.bg.py - Update config.json: YTDLP_* as primary keys, MEDIA_* as x-aliases - Update templates CSS classes: media-* → ytdlp-* - Fix gallerydl bug: remove incorrect dependency on media plugin output - Update all codebase references to use YTDLP_* and SAVE_YTDLP - Add backwards compatibility test for MEDIA_ENABLED alias
2026-04-04 23:07:56 +10:00 · 2025-12-29 19:09:05 +00:00
parent 30c60eef76
commit a5654e877f
19 changed files with 110 additions and 102 deletions
--- a/archivebox/cli/tests_piping.py
+++ b/archivebox/cli/tests_piping.py
@@ -44,7 +44,7 @@ TEST_CONFIG = {
    'SAVE_READABILITY': 'False',
    'SAVE_MERCURY': 'False',
    'SAVE_GIT': 'False',
-    'SAVE_MEDIA': 'False',
+    'SAVE_YTDLP': 'False',
    'SAVE_HEADERS': 'False',
    'USE_CURL': 'False',
    'USE_WGET': 'False',
--- a/archivebox/plugins/gallerydl/on_Snapshot__64_gallerydl.bg.py
+++ b/archivebox/plugins/gallerydl/on_Snapshot__64_gallerydl.bg.py
@@ -59,7 +59,6 @@ def get_env_int(name: str, default: int = 0) -> int:


 STATICFILE_DIR = '../staticfile'
-MEDIA_DIR = '../media'

 def has_staticfile_output() -> bool:
    """Check if staticfile extractor already downloaded this URL."""
@@ -67,12 +66,6 @@ def has_staticfile_output() -> bool:
    return staticfile_dir.exists() and any(staticfile_dir.iterdir())


-def has_media_output() -> bool:
-    """Check if media extractor already downloaded this URL."""
-    media_dir = Path(MEDIA_DIR)
-    return media_dir.exists() and any(media_dir.iterdir())
-
-
 # Default gallery-dl args
 def get_gallerydl_default_args() -> list[str]:
    """Build default gallery-dl arguments."""
@@ -185,7 +178,7 @@ def main(url: str, snapshot_id: str):
            # Temporary failure (config disabled) - NO JSONL emission
            sys.exit(0)

-        # Check if staticfile or media extractors already handled this (permanent skip)
+        # Check if staticfile extractor already handled this (permanent skip)
        if has_staticfile_output():
            print(f'Skipping gallery-dl - staticfile extractor already downloaded this', file=sys.stderr)
            print(json.dumps({
@@ -195,15 +188,6 @@ def main(url: str, snapshot_id: str):
            }))
            sys.exit(0)

-        if has_media_output():
-            print(f'Skipping gallery-dl - media extractor already downloaded this', file=sys.stderr)
-            print(json.dumps({
-                'type': 'ArchiveResult',
-                'status': 'skipped',
-                'output_str': 'media already handled',
-            }))
-            sys.exit(0)
-
        # Get binary from environment
        binary = get_env('GALLERYDL_BINARY', 'gallery-dl')

--- a/archivebox/plugins/ytdlp/binaries.jsonl
+++ b/archivebox/plugins/ytdlp/binaries.jsonl
--- a/archivebox/plugins/ytdlp/config.json
+++ b/archivebox/plugins/ytdlp/config.json
@@ -3,36 +3,38 @@
  "type": "object",
  "additionalProperties": false,
  "properties": {
-    "MEDIA_ENABLED": {
+    "YTDLP_ENABLED": {
      "type": "boolean",
      "default": true,
-      "x-aliases": ["SAVE_MEDIA", "USE_MEDIA", "USE_YTDLP", "FETCH_MEDIA"],
-      "description": "Enable media downloading with yt-dlp"
+      "x-aliases": ["MEDIA_ENABLED", "SAVE_MEDIA", "USE_MEDIA", "USE_YTDLP", "FETCH_MEDIA", "SAVE_YTDLP"],
+      "description": "Enable video/audio downloading with yt-dlp"
    },
    "YTDLP_BINARY": {
      "type": "string",
      "default": "yt-dlp",
-      "x-aliases": ["YOUTUBEDL_BINARY", "YOUTUBE_DL_BINARY", "MEDIA_BINARY"],
+      "x-aliases": ["MEDIA_BINARY", "YOUTUBEDL_BINARY", "YOUTUBE_DL_BINARY"],
      "description": "Path to yt-dlp binary"
    },
-    "MEDIA_TIMEOUT": {
+    "YTDLP_TIMEOUT": {
      "type": "integer",
      "default": 3600,
      "minimum": 30,
      "x-fallback": "TIMEOUT",
-      "description": "Timeout for media downloads in seconds"
+      "x-aliases": ["MEDIA_TIMEOUT"],
+      "description": "Timeout for yt-dlp downloads in seconds"
    },
-    "MEDIA_MAX_SIZE": {
+    "YTDLP_MAX_SIZE": {
      "type": "string",
      "default": "750m",
      "pattern": "^\\d+[kmgKMG]?$",
-      "description": "Maximum file size for media downloads"
+      "x-aliases": ["MEDIA_MAX_SIZE"],
+      "description": "Maximum file size for yt-dlp downloads"
    },
-    "MEDIA_CHECK_SSL_VALIDITY": {
+    "YTDLP_CHECK_SSL_VALIDITY": {
      "type": "boolean",
      "default": true,
      "x-fallback": "CHECK_SSL_VALIDITY",
-      "x-aliases": ["YTDLP_CHECK_SSL_VALIDITY"],
+      "x-aliases": ["MEDIA_CHECK_SSL_VALIDITY"],
      "description": "Whether to verify SSL certificates"
    },
    "YTDLP_ARGS": {
--- a/archivebox/plugins/ytdlp/on_Snapshot__63_ytdlp.bg.py
+++ b/archivebox/plugins/ytdlp/on_Snapshot__63_ytdlp.bg.py
@@ -1,25 +1,23 @@
 #!/usr/bin/env python3
 """
-Download media from a URL using yt-dlp.
+Download video/audio from a URL using yt-dlp.

-Usage: on_Snapshot__media.py --url=<url> --snapshot-id=<uuid>
-Output: Downloads media files to $PWD/media/
+Usage: on_Snapshot__ytdlp.py --url=<url> --snapshot-id=<uuid>
+Output: Downloads video/audio files to $PWD/ytdlp/

 Environment variables:
    YTDLP_BINARY: Path to yt-dlp binary
-    YTDLP_TIMEOUT: Timeout in seconds (default: 3600 for large media)
+    YTDLP_TIMEOUT: Timeout in seconds (default: 3600 for large downloads)
    YTDLP_CHECK_SSL_VALIDITY: Whether to check SSL certificates (default: True)
    YTDLP_EXTRA_ARGS: Extra arguments for yt-dlp (space-separated)
+    YTDLP_MAX_SIZE: Maximum file size (default: 750m)

-    # Media feature toggles
-    USE_YTDLP: Enable yt-dlp media extraction (default: True)
-    SAVE_MEDIA: Alias for USE_YTDLP
-
-    # Media size limits
-    MEDIA_MAX_SIZE: Maximum media file size (default: 750m)
+    # Feature toggles (with backwards-compatible aliases)
+    YTDLP_ENABLED: Enable yt-dlp extraction (default: True)
+    SAVE_YTDLP: Alias for YTDLP_ENABLED
+    MEDIA_ENABLED: Backwards-compatible alias for YTDLP_ENABLED

    # Fallback to ARCHIVING_CONFIG values if YTDLP_* not set:
-    MEDIA_TIMEOUT: Fallback timeout for media
    TIMEOUT: Fallback timeout
    CHECK_SSL_VALIDITY: Fallback SSL check
 """
@@ -34,7 +32,7 @@ import rich_click as click


 # Extractor metadata
-PLUGIN_NAME = 'media'
+PLUGIN_NAME = 'ytdlp'
 BIN_NAME = 'yt-dlp'
 BIN_PROVIDERS = 'pip,apt,brew,env'
 OUTPUT_DIR = '.'
@@ -69,7 +67,7 @@ def has_staticfile_output() -> bool:


 # Default yt-dlp args (from old YTDLP_CONFIG)
-def get_ytdlp_default_args(media_max_size: str = '750m') -> list[str]:
+def get_ytdlp_default_args(max_size: str = '750m') -> list[str]:
    """Build default yt-dlp arguments."""
    return [
        '--restrict-filenames',
@@ -86,21 +84,21 @@ def get_ytdlp_default_args(media_max_size: str = '750m') -> list[str]:
        '--ignore-errors',
        '--geo-bypass',
        '--add-metadata',
-        f'--format=(bv*+ba/b)[filesize<={media_max_size}][filesize_approx<=?{media_max_size}]/(bv*+ba/b)',
+        f'--format=(bv*+ba/b)[filesize<={max_size}][filesize_approx<=?{max_size}]/(bv*+ba/b)',
    ]


-def save_media(url: str, binary: str) -> tuple[bool, str | None, str]:
+def save_ytdlp(url: str, binary: str) -> tuple[bool, str | None, str]:
    """
-    Download media using yt-dlp.
+    Download video/audio using yt-dlp.

    Returns: (success, output_path, error_message)
    """
-    # Get config from env
+    # Get config from env (YTDLP_* primary, MEDIA_* as fallback via aliases)
    timeout = get_env_int('TIMEOUT', 3600)
    check_ssl = get_env_bool('CHECK_SSL_VALIDITY', True)
    extra_args = get_env('YTDLP_EXTRA_ARGS', '')
-    media_max_size = get_env('MEDIA_MAX_SIZE', '750m')
+    max_size = get_env('YTDLP_MAX_SIZE', '') or get_env('MEDIA_MAX_SIZE', '750m')

    # Output directory is current directory (hook already runs in output dir)
    output_dir = Path(OUTPUT_DIR)
@@ -108,7 +106,7 @@ def save_media(url: str, binary: str) -> tuple[bool, str | None, str]:
    # Build command (later options take precedence)
    cmd = [
        binary,
-        *get_ytdlp_default_args(media_max_size),
+        *get_ytdlp_default_args(max_size),
        '--no-progress',
        '-o', '%(title)s.%(ext)s',
    ]
@@ -175,21 +173,22 @@ def save_media(url: str, binary: str) -> tuple[bool, str | None, str]:


@click.command()
-@click.option('--url', required=True, help='URL to download media from')
+@click.option('--url', required=True, help='URL to download video/audio from')
@click.option('--snapshot-id', required=True, help='Snapshot UUID')
 def main(url: str, snapshot_id: str):
-    """Download media from a URL using yt-dlp."""
+    """Download video/audio from a URL using yt-dlp."""

    try:
-        # Check if media downloading is enabled
-        if not get_env_bool('MEDIA_ENABLED', True):
-            print('Skipping media (MEDIA_ENABLED=False)', file=sys.stderr)
+        # Check if yt-dlp downloading is enabled (YTDLP_ENABLED primary, MEDIA_ENABLED fallback)
+        ytdlp_enabled = get_env_bool('YTDLP_ENABLED', True) and get_env_bool('MEDIA_ENABLED', True)
+        if not ytdlp_enabled:
+            print('Skipping ytdlp (YTDLP_ENABLED=False)', file=sys.stderr)
            # Temporary failure (config disabled) - NO JSONL emission
            sys.exit(0)

        # Check if staticfile extractor already handled this (permanent skip)
        if has_staticfile_output():
-            print('Skipping media - staticfile extractor already downloaded this', file=sys.stderr)
+            print('Skipping ytdlp - staticfile extractor already downloaded this', file=sys.stderr)
            print(json.dumps({'type': 'ArchiveResult', 'status': 'skipped', 'output_str': 'staticfile already exists'}))
            sys.exit(0)

@@ -197,7 +196,7 @@ def main(url: str, snapshot_id: str):
        binary = get_env('YTDLP_BINARY', 'yt-dlp')

        # Run extraction
-        success, output, error = save_media(url, binary)
+        success, output, error = save_ytdlp(url, binary)

        if success:
            # Success - emit ArchiveResult
--- a/archivebox/plugins/ytdlp/templates/embed.html
+++ b/archivebox/plugins/ytdlp/templates/embed.html
@@ -1,5 +1,5 @@
-<!-- Media embed - video/audio player -->
-<div class="extractor-embed media-embed" style="width: 100%; height: 100%; min-height: 400px; background: #1a1a1a; display: flex; align-items: center; justify-content: center;">
+<!-- YT-DLP embed - video/audio player -->
+<div class="extractor-embed ytdlp-embed" style="width: 100%; height: 100%; min-height: 400px; background: #1a1a1a; display: flex; align-items: center; justify-content: center;">
    <video src="{{ output_path }}"
           style="max-width: 100%; max-height: 100%;"
           controls
--- a/archivebox/plugins/ytdlp/templates/fullscreen.html
+++ b/archivebox/plugins/ytdlp/templates/fullscreen.html
@@ -1,5 +1,5 @@
-<!-- Media fullscreen - full video/audio player -->
-<div class="extractor-fullscreen media-fullscreen" style="width: 100%; height: 100vh; background: #000; display: flex; align-items: center; justify-content: center;">
+<!-- YT-DLP fullscreen - full video/audio player -->
+<div class="extractor-fullscreen ytdlp-fullscreen" style="width: 100%; height: 100vh; background: #000; display: flex; align-items: center; justify-content: center;">
    <video src="{{ output_path }}"
           style="max-width: 100%; max-height: 100%;"
           controls
--- a/archivebox/plugins/ytdlp/templates/icon.html
+++ b/archivebox/plugins/ytdlp/templates/icon.html
--- a/archivebox/plugins/ytdlp/templates/thumbnail.html
+++ b/archivebox/plugins/ytdlp/templates/thumbnail.html
@@ -1,5 +1,5 @@
-<!-- Media thumbnail - shows video/audio player or placeholder -->
-<div class="extractor-thumbnail media-thumbnail" style="width: 100%; height: 100px; overflow: hidden; background: #1a1a1a; display: flex; align-items: center; justify-content: center;">
+<!-- YT-DLP thumbnail - shows video/audio player or placeholder -->
+<div class="extractor-thumbnail ytdlp-thumbnail" style="width: 100%; height: 100px; overflow: hidden; background: #1a1a1a; display: flex; align-items: center; justify-content: center;">
    <video src="{{ output_path }}"
           style="width: 100%; height: 100px; object-fit: contain;"
           poster=""
@@ -9,6 +9,6 @@
    </video>
    <div style="display: none; flex-direction: column; align-items: center; color: #888; font-size: 12px;">
        <span style="font-size: 32px;">🎬</span>
-        <span>Media</span>
+        <span>YT-DLP</span>
    </div>
 </div>
--- a/archivebox/plugins/ytdlp/tests/test_ytdlp.py
+++ b/archivebox/plugins/ytdlp/tests/test_ytdlp.py
@@ -1,15 +1,14 @@
 """
-Integration tests for media plugin
+Integration tests for ytdlp plugin

 Tests verify:
-    pass
 1. Hook script exists
 2. Dependencies installed via validation hooks
 3. Verify deps with abx-pkg
-4. Media extraction works on video URLs
+4. YT-DLP extraction works on video URLs
 5. JSONL output is correct
-6. Config options work
-7. Handles non-media URLs gracefully
+6. Config options work (YTDLP_* and backwards-compatible MEDIA_* aliases)
+7. Handles non-video URLs gracefully
 """

 import json
@@ -22,12 +21,12 @@ import pytest

 PLUGIN_DIR = Path(__file__).parent.parent
 PLUGINS_ROOT = PLUGIN_DIR.parent
-MEDIA_HOOK = next(PLUGIN_DIR.glob('on_Snapshot__*_media.*'), None)
+YTDLP_HOOK = next(PLUGIN_DIR.glob('on_Snapshot__*_ytdlp.*'), None)
 TEST_URL = 'https://example.com/video.mp4'

 def test_hook_script_exists():
    """Verify on_Snapshot hook exists."""
-    assert MEDIA_HOOK.exists(), f"Hook not found: {MEDIA_HOOK}"
+    assert YTDLP_HOOK.exists(), f"Hook not found: {YTDLP_HOOK}"


 def test_verify_deps_with_abx_pkg():
@@ -60,16 +59,16 @@ def test_verify_deps_with_abx_pkg():
    if missing_binaries:
        pass

-def test_handles_non_media_url():
-    """Test that media extractor handles non-media URLs gracefully via hook."""
+def test_handles_non_video_url():
+    """Test that ytdlp extractor handles non-video URLs gracefully via hook."""
    # Prerequisites checked by earlier test

    with tempfile.TemporaryDirectory() as tmpdir:
        tmpdir = Path(tmpdir)

-        # Run media extraction hook on non-media URL
+        # Run ytdlp extraction hook on non-video URL
        result = subprocess.run(
-            [sys.executable, str(MEDIA_HOOK), '--url', 'https://example.com', '--snapshot-id', 'test789'],
+            [sys.executable, str(YTDLP_HOOK), '--url', 'https://example.com', '--snapshot-id', 'test789'],
            cwd=tmpdir,
            capture_output=True,
            text=True,
@@ -97,16 +96,16 @@ def test_handles_non_media_url():
        assert result_json['status'] == 'succeeded', f"Should succeed: {result_json}"


-def test_config_save_media_false_skips():
-    """Test that MEDIA_ENABLED=False exits without emitting JSONL."""
+def test_config_ytdlp_enabled_false_skips():
+    """Test that YTDLP_ENABLED=False exits without emitting JSONL."""
    import os

    with tempfile.TemporaryDirectory() as tmpdir:
        env = os.environ.copy()
-        env['MEDIA_ENABLED'] = 'False'
+        env['YTDLP_ENABLED'] = 'False'

        result = subprocess.run(
-            [sys.executable, str(MEDIA_HOOK), '--url', TEST_URL, '--snapshot-id', 'test999'],
+            [sys.executable, str(YTDLP_HOOK), '--url', TEST_URL, '--snapshot-id', 'test999'],
            cwd=tmpdir,
            capture_output=True,
            text=True,
@@ -124,17 +123,41 @@ def test_config_save_media_false_skips():
        assert len(jsonl_lines) == 0, f"Should not emit JSONL when feature disabled, but got: {jsonl_lines}"


-def test_config_timeout():
-    """Test that MEDIA_TIMEOUT config is respected."""
+def test_config_media_enabled_backwards_compat():
+    """Test that MEDIA_ENABLED=False (backwards-compatible alias) also works."""
    import os

    with tempfile.TemporaryDirectory() as tmpdir:
        env = os.environ.copy()
-        env['MEDIA_TIMEOUT'] = '5'
+        env['MEDIA_ENABLED'] = 'False'
+
+        result = subprocess.run(
+            [sys.executable, str(YTDLP_HOOK), '--url', TEST_URL, '--snapshot-id', 'test_compat'],
+            cwd=tmpdir,
+            capture_output=True,
+            text=True,
+            env=env,
+            timeout=30
+        )
+
+        assert result.returncode == 0, f"Should exit 0 when feature disabled via MEDIA_ENABLED: {result.stderr}"
+
+        # Should NOT emit any JSONL when disabled via backwards-compatible alias
+        jsonl_lines = [line for line in result.stdout.strip().split('\n') if line.strip().startswith('{')]
+        assert len(jsonl_lines) == 0, f"Should not emit JSONL when feature disabled via MEDIA_ENABLED, but got: {jsonl_lines}"
+
+
+def test_config_timeout():
+    """Test that YTDLP_TIMEOUT config is respected (also via MEDIA_TIMEOUT alias)."""
+    import os
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        env = os.environ.copy()
+        env['YTDLP_TIMEOUT'] = '5'

        start_time = time.time()
        result = subprocess.run(
-            [sys.executable, str(MEDIA_HOOK), '--url', 'https://example.com', '--snapshot-id', 'testtimeout'],
+            [sys.executable, str(YTDLP_HOOK), '--url', 'https://example.com', '--snapshot-id', 'testtimeout'],
            cwd=tmpdir,
            capture_output=True,
            text=True,
@@ -149,7 +172,7 @@ def test_config_timeout():


 def test_real_youtube_url():
-    """Test that yt-dlp can extract media from a real YouTube URL."""
+    """Test that yt-dlp can extract video/audio from a real YouTube URL."""
    import os

    with tempfile.TemporaryDirectory() as tmpdir:
@@ -159,11 +182,11 @@ def test_real_youtube_url():
        youtube_url = 'https://www.youtube.com/watch?v=jNQXAC9IVRw'  # "Me at the zoo" - first YouTube video

        env = os.environ.copy()
-        env['MEDIA_TIMEOUT'] = '120'  # Give it time to download
+        env['YTDLP_TIMEOUT'] = '120'  # Give it time to download

        start_time = time.time()
        result = subprocess.run(
-            [sys.executable, str(MEDIA_HOOK), '--url', youtube_url, '--snapshot-id', 'testyoutube'],
+            [sys.executable, str(YTDLP_HOOK), '--url', youtube_url, '--snapshot-id', 'testyoutube'],
            cwd=tmpdir,
            capture_output=True,
            text=True,
@@ -173,7 +196,7 @@ def test_real_youtube_url():
        elapsed_time = time.time() - start_time

        # Should succeed
-        assert result.returncode == 0, f"Should extract media successfully: {result.stderr}"
+        assert result.returncode == 0, f"Should extract video/audio successfully: {result.stderr}"

        # Parse JSONL output
        result_json = None
@@ -191,11 +214,11 @@ def test_real_youtube_url():
        assert result_json, f"Should have ArchiveResult JSONL output. stdout: {result.stdout}"
        assert result_json['status'] == 'succeeded', f"Should succeed: {result_json}"

-        # Check that some media files were downloaded
+        # Check that some video/audio files were downloaded
        output_files = list(tmpdir.glob('**/*'))
        media_files = [f for f in output_files if f.is_file() and f.suffix.lower() in ('.mp4', '.webm', '.mkv', '.m4a', '.mp3', '.json', '.jpg', '.webp')]

-        assert len(media_files) > 0, f"Should have downloaded at least one media file. Files: {output_files}"
+        assert len(media_files) > 0, f"Should have downloaded at least one video/audio file. Files: {output_files}"

        print(f"Successfully extracted {len(media_files)} file(s) in {elapsed_time:.2f}s")

--- a/archivebox/tests/test_migrations_08_to_09.py
+++ b/archivebox/tests/test_migrations_08_to_09.py
@@ -458,7 +458,7 @@ class TestFilesystemMigration08to09(unittest.TestCase):
                    'SAVE_READABILITY': 'True',
                    'SAVE_MERCURY': 'True',
                    'SAVE_PDF': 'True',
-                    'SAVE_MEDIA': 'True',
+                    'SAVE_YTDLP': 'True',
                    'SAVE_ARCHIVEDOTORG': 'True',
                    'SAVE_HEADERS': 'True',
                    'SAVE_HTMLTOTEXT': 'True',
--- a/archivebox/tests/test_migrations_helpers.py
+++ b/archivebox/tests/test_migrations_helpers.py
@@ -1002,7 +1002,7 @@ def run_archivebox(data_dir: Path, args: list, timeout: int = 60, env: dict = No
    base_env['SAVE_READABILITY'] = 'False'
    base_env['SAVE_MERCURY'] = 'False'
    base_env['SAVE_GIT'] = 'False'
-    base_env['SAVE_MEDIA'] = 'False'
+    base_env['SAVE_YTDLP'] = 'False'
    base_env['SAVE_HEADERS'] = 'False'
    base_env['SAVE_HTMLTOTEXT'] = 'False'