Rename media plugin to ytdlp (#1722)

- Rename archivebox/plugins/media/ → archivebox/plugins/ytdlp/
- Rename hook script on_Snapshot__63_media.bg.py →
on_Snapshot__63_ytdlp.bg.py
- Update config.json: YTDLP_* as primary keys, MEDIA_* as x-aliases
- Update templates CSS classes: media-* → ytdlp-*
- Fix gallerydl bug: remove incorrect dependency on media plugin output
- Update all codebase references to use YTDLP_* and SAVE_YTDLP
- Add backwards compatibility test for MEDIA_ENABLED alias

<!-- IMPORTANT: Do not submit PRs with only formatting / PEP8 / line
length changes. -->

# Summary

<!--e.g. This PR fixes ABC or adds the ability to do XYZ...-->

# Related issues

<!-- e.g. #123 or Roadmap goal #
https://github.com/pirate/ArchiveBox/wiki/Roadmap -->

# Changes these areas

- [ ] Bugfixes
- [ ] Feature behavior
- [ ] Command line interface
- [ ] Configuration options
- [ ] Internal architecture
- [ ] Snapshot data layout on disk
This commit is contained in:
Nick Sweeting
2025-12-29 11:47:05 -08:00
committed by GitHub
20 changed files with 205 additions and 172 deletions

View File

@@ -764,7 +764,7 @@ The configuration is documented here: **[Configuration Wiki](https://github.com/
TIMEOUT=240 # default: 60 add more seconds on slower networks
CHECK_SSL_VALIDITY=False # default: True False = allow saving URLs w/ bad SSL
SAVE_ARCHIVEDOTORG=False # default: True False = disable Archive.org saving
MAX_MEDIA_SIZE=1500m # default: 750m raise/lower youtubedl output size
YTDLP_MAX_SIZE=1500m # default: 750m raise/lower yt-dlp output size
<br/>
PUBLIC_INDEX=True # default: True whether anon users can view index
PUBLIC_SNAPSHOTS=True # default: True whether anon users can view pages
@@ -1089,7 +1089,7 @@ Because ArchiveBox is designed to ingest a large volume of URLs with multiple co
<br/>
<ul>
<li><strong>ArchiveBox can use anywhere from ~1gb per 1000 Snapshots, to ~50gb per 1000 Snapshots</strong>, mostly dependent on whether you're saving audio & video using <code>SAVE_MEDIA=True</code> and whether you lower <code>MEDIA_MAX_SIZE=750mb</code>.</li>
<li><strong>ArchiveBox can use anywhere from ~1gb per 1000 Snapshots, to ~50gb per 1000 Snapshots</strong>, mostly dependent on whether you're saving video/audio using <code>YTDLP_ENABLED=True</code> and whether you lower <code>YTDLP_MAX_SIZE=750m</code>.</li>
<li>Disk usage can be reduced by using a compressed/<a href="https://www.ixsystems.com/blog/ixsystems-and-klara-systems-celebrate-valentines-day-with-a-heartfelt-donation-of-fast-dedupe-to-openzfs-and-truenas/">deduplicated</a> filesystem like <a href="https://www.reddit.com/r/zfs/comments/t9cexx/a_simple_real_world_zfs_compression_speed_an/">ZFS</a>/BTRFS, or by turning off extractors methods you don't need. You can also deduplicate content with a tool like <a href="https://github.com/adrianlopezroche/fdupes"><code>fdupes</code></a> or <a href="https://github.com/pauldreik/rdfind"><code>rdfind</code></a>.
</li>
<li><strong>Don't store large collections on older filesystems like EXT3/FAT</strong> as they may not be able to handle more than 50k directory entries in the <code>data/archive/</code> folder.

View File

@@ -451,12 +451,12 @@ except subprocess.TimeoutExpired:
### Background Hook (Long-Running Download)
```python
#!/usr/bin/env python3
# archivebox/plugins/media/on_Snapshot__63_media.bg.py
# archivebox/plugins/ytdlp/on_Snapshot__63_ytdlp.bg.py
# Runs at step 6, doesn't block step progression
# Gets full MEDIA_TIMEOUT (e.g., 3600s) regardless of when step 99 completes
# Gets full YTDLP_TIMEOUT (e.g., 3600s) regardless of when step 99 completes
timeout = get_env_int('YTDLP_TIMEOUT') or get_env_int('MEDIA_TIMEOUT') or get_env_int('TIMEOUT', 3600)
timeout = get_env_int('YTDLP_TIMEOUT') or get_env_int('TIMEOUT', 3600)
try:
result = subprocess.run(['yt-dlp', url], capture_output=True, timeout=timeout)

View File

@@ -44,7 +44,7 @@ TEST_CONFIG = {
'SAVE_READABILITY': 'False',
'SAVE_MERCURY': 'False',
'SAVE_GIT': 'False',
'SAVE_MEDIA': 'False',
'SAVE_YTDLP': 'False',
'SAVE_HEADERS': 'False',
'USE_CURL': 'False',
'USE_WGET': 'False',

View File

@@ -59,7 +59,6 @@ def get_env_int(name: str, default: int = 0) -> int:
STATICFILE_DIR = '../staticfile'
MEDIA_DIR = '../media'
def has_staticfile_output() -> bool:
"""Check if staticfile extractor already downloaded this URL."""
@@ -67,12 +66,6 @@ def has_staticfile_output() -> bool:
return staticfile_dir.exists() and any(staticfile_dir.iterdir())
def has_media_output() -> bool:
"""Check if media extractor already downloaded this URL."""
media_dir = Path(MEDIA_DIR)
return media_dir.exists() and any(media_dir.iterdir())
# Default gallery-dl args
def get_gallerydl_default_args() -> list[str]:
"""Build default gallery-dl arguments."""
@@ -185,7 +178,7 @@ def main(url: str, snapshot_id: str):
# Temporary failure (config disabled) - NO JSONL emission
sys.exit(0)
# Check if staticfile or media extractors already handled this (permanent skip)
# Check if staticfile extractor already handled this (permanent skip)
if has_staticfile_output():
print(f'Skipping gallery-dl - staticfile extractor already downloaded this', file=sys.stderr)
print(json.dumps({
@@ -195,15 +188,6 @@ def main(url: str, snapshot_id: str):
}))
sys.exit(0)
if has_media_output():
print(f'Skipping gallery-dl - media extractor already downloaded this', file=sys.stderr)
print(json.dumps({
'type': 'ArchiveResult',
'status': 'skipped',
'output_str': 'media already handled',
}))
sys.exit(0)
# Get binary from environment
binary = get_env('GALLERYDL_BINARY', 'gallery-dl')

View File

@@ -1,58 +0,0 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"additionalProperties": false,
"properties": {
"MEDIA_ENABLED": {
"type": "boolean",
"default": true,
"x-aliases": ["SAVE_MEDIA", "USE_MEDIA", "USE_YTDLP", "FETCH_MEDIA"],
"description": "Enable media downloading with yt-dlp"
},
"YTDLP_BINARY": {
"type": "string",
"default": "yt-dlp",
"x-aliases": ["YOUTUBEDL_BINARY", "YOUTUBE_DL_BINARY", "MEDIA_BINARY"],
"description": "Path to yt-dlp binary"
},
"MEDIA_TIMEOUT": {
"type": "integer",
"default": 3600,
"minimum": 30,
"x-fallback": "TIMEOUT",
"description": "Timeout for media downloads in seconds"
},
"MEDIA_MAX_SIZE": {
"type": "string",
"default": "750m",
"pattern": "^\\d+[kmgKMG]?$",
"description": "Maximum file size for media downloads"
},
"MEDIA_CHECK_SSL_VALIDITY": {
"type": "boolean",
"default": true,
"x-fallback": "CHECK_SSL_VALIDITY",
"x-aliases": ["YTDLP_CHECK_SSL_VALIDITY"],
"description": "Whether to verify SSL certificates"
},
"YTDLP_ARGS": {
"type": "array",
"items": {"type": "string"},
"default": [
"--write-info-json",
"--write-thumbnail",
"--write-sub",
"--embed-subs",
"--write-auto-sub"
],
"x-aliases": ["MEDIA_ARGS"],
"description": "Default yt-dlp arguments"
},
"YTDLP_EXTRA_ARGS": {
"type": "string",
"default": "",
"x-aliases": ["MEDIA_EXTRA_ARGS"],
"description": "Extra arguments for yt-dlp (space-separated)"
}
}
}

View File

@@ -0,0 +1,71 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"type": "object",
"additionalProperties": false,
"properties": {
"YTDLP_ENABLED": {
"type": "boolean",
"default": true,
"x-aliases": ["MEDIA_ENABLED", "SAVE_MEDIA", "USE_MEDIA", "USE_YTDLP", "FETCH_MEDIA", "SAVE_YTDLP"],
"description": "Enable video/audio downloading with yt-dlp"
},
"YTDLP_BINARY": {
"type": "string",
"default": "yt-dlp",
"x-aliases": ["MEDIA_BINARY", "YOUTUBEDL_BINARY", "YOUTUBE_DL_BINARY"],
"description": "Path to yt-dlp binary"
},
"YTDLP_TIMEOUT": {
"type": "integer",
"default": 3600,
"minimum": 30,
"x-fallback": "TIMEOUT",
"x-aliases": ["MEDIA_TIMEOUT"],
"description": "Timeout for yt-dlp downloads in seconds"
},
"YTDLP_MAX_SIZE": {
"type": "string",
"default": "750m",
"pattern": "^\\d+[kmgKMG]?$",
"x-aliases": ["MEDIA_MAX_SIZE"],
"description": "Maximum file size for yt-dlp downloads"
},
"YTDLP_CHECK_SSL_VALIDITY": {
"type": "boolean",
"default": true,
"x-fallback": "CHECK_SSL_VALIDITY",
"x-aliases": ["MEDIA_CHECK_SSL_VALIDITY"],
"description": "Whether to verify SSL certificates"
},
"YTDLP_ARGS": {
"type": "array",
"items": {"type": "string"},
"default": [
"--restrict-filenames",
"--trim-filenames", "128",
"--write-description",
"--write-info-json",
"--write-thumbnail",
"--write-sub",
"--write-auto-subs",
"--convert-subs=srt",
"--yes-playlist",
"--continue",
"--no-abort-on-error",
"--ignore-errors",
"--geo-bypass",
"--add-metadata",
"--no-progress",
"-o", "%(title)s.%(ext)s"
],
"x-aliases": ["MEDIA_ARGS"],
"description": "Default yt-dlp arguments (override to customize behavior)"
},
"YTDLP_EXTRA_ARGS": {
"type": "string",
"default": "",
"x-aliases": ["MEDIA_EXTRA_ARGS"],
"description": "Extra arguments for yt-dlp (space-separated, appended after YTDLP_ARGS)"
}
}
}

View File

@@ -1,25 +1,24 @@
#!/usr/bin/env python3
"""
Download media from a URL using yt-dlp.
Download video/audio from a URL using yt-dlp.
Usage: on_Snapshot__media.py --url=<url> --snapshot-id=<uuid>
Output: Downloads media files to $PWD/media/
Usage: on_Snapshot__ytdlp.py --url=<url> --snapshot-id=<uuid>
Output: Downloads video/audio files to $PWD/ytdlp/
Environment variables:
YTDLP_BINARY: Path to yt-dlp binary
YTDLP_TIMEOUT: Timeout in seconds (default: 3600 for large media)
YTDLP_TIMEOUT: Timeout in seconds (default: 3600 for large downloads)
YTDLP_CHECK_SSL_VALIDITY: Whether to check SSL certificates (default: True)
YTDLP_EXTRA_ARGS: Extra arguments for yt-dlp (space-separated)
YTDLP_ARGS: JSON array of yt-dlp arguments (overrides defaults)
YTDLP_EXTRA_ARGS: Extra arguments for yt-dlp (space-separated, appended)
YTDLP_MAX_SIZE: Maximum file size (default: 750m)
# Media feature toggles
USE_YTDLP: Enable yt-dlp media extraction (default: True)
SAVE_MEDIA: Alias for USE_YTDLP
# Media size limits
MEDIA_MAX_SIZE: Maximum media file size (default: 750m)
# Feature toggles (with backwards-compatible aliases)
YTDLP_ENABLED: Enable yt-dlp extraction (default: True)
SAVE_YTDLP: Alias for YTDLP_ENABLED
MEDIA_ENABLED: Backwards-compatible alias for YTDLP_ENABLED
# Fallback to ARCHIVING_CONFIG values if YTDLP_* not set:
MEDIA_TIMEOUT: Fallback timeout for media
TIMEOUT: Fallback timeout
CHECK_SSL_VALIDITY: Fallback SSL check
"""
@@ -34,7 +33,7 @@ import rich_click as click
# Extractor metadata
PLUGIN_NAME = 'media'
PLUGIN_NAME = 'ytdlp'
BIN_NAME = 'yt-dlp'
BIN_PROVIDERS = 'pip,apt,brew,env'
OUTPUT_DIR = '.'
@@ -68,49 +67,62 @@ def has_staticfile_output() -> bool:
return staticfile_dir.exists() and any(staticfile_dir.iterdir())
# Default yt-dlp args (from old YTDLP_CONFIG)
def get_ytdlp_default_args(media_max_size: str = '750m') -> list[str]:
"""Build default yt-dlp arguments."""
return [
'--restrict-filenames',
'--trim-filenames', '128',
'--write-description',
'--write-info-json',
'--write-thumbnail',
'--write-sub',
'--write-auto-subs',
'--convert-subs=srt',
'--yes-playlist',
'--continue',
'--no-abort-on-error',
'--ignore-errors',
'--geo-bypass',
'--add-metadata',
f'--format=(bv*+ba/b)[filesize<={media_max_size}][filesize_approx<=?{media_max_size}]/(bv*+ba/b)',
]
# Default yt-dlp args (can be overridden via YTDLP_ARGS env var)
YTDLP_DEFAULT_ARGS = [
'--restrict-filenames',
'--trim-filenames', '128',
'--write-description',
'--write-info-json',
'--write-thumbnail',
'--write-sub',
'--write-auto-subs',
'--convert-subs=srt',
'--yes-playlist',
'--continue',
'--no-abort-on-error',
'--ignore-errors',
'--geo-bypass',
'--add-metadata',
'--no-progress',
'-o', '%(title)s.%(ext)s',
]
def save_media(url: str, binary: str) -> tuple[bool, str | None, str]:
def get_ytdlp_args() -> list[str]:
"""Get yt-dlp arguments from YTDLP_ARGS env var or use defaults."""
ytdlp_args_str = get_env('YTDLP_ARGS', '')
if ytdlp_args_str:
try:
# Try to parse as JSON array
args = json.loads(ytdlp_args_str)
if isinstance(args, list):
return [str(arg) for arg in args]
except json.JSONDecodeError:
pass
return YTDLP_DEFAULT_ARGS
def save_ytdlp(url: str, binary: str) -> tuple[bool, str | None, str]:
"""
Download media using yt-dlp.
Download video/audio using yt-dlp.
Returns: (success, output_path, error_message)
"""
# Get config from env
# Get config from env (YTDLP_* primary, MEDIA_* as fallback via aliases)
timeout = get_env_int('TIMEOUT', 3600)
check_ssl = get_env_bool('CHECK_SSL_VALIDITY', True)
extra_args = get_env('YTDLP_EXTRA_ARGS', '')
media_max_size = get_env('MEDIA_MAX_SIZE', '750m')
max_size = get_env('YTDLP_MAX_SIZE', '') or get_env('MEDIA_MAX_SIZE', '750m')
# Output directory is current directory (hook already runs in output dir)
output_dir = Path(OUTPUT_DIR)
# Build command (later options take precedence)
# Build command using configurable YTDLP_ARGS (later options take precedence)
cmd = [
binary,
*get_ytdlp_default_args(media_max_size),
'--no-progress',
'-o', '%(title)s.%(ext)s',
*get_ytdlp_args(),
# Format with max_size limit (appended after YTDLP_ARGS so it can be overridden by YTDLP_EXTRA_ARGS)
f'--format=(bv*+ba/b)[filesize<={max_size}][filesize_approx<=?{max_size}]/(bv*+ba/b)',
]
if not check_ssl:
@@ -175,21 +187,22 @@ def save_media(url: str, binary: str) -> tuple[bool, str | None, str]:
@click.command()
@click.option('--url', required=True, help='URL to download media from')
@click.option('--url', required=True, help='URL to download video/audio from')
@click.option('--snapshot-id', required=True, help='Snapshot UUID')
def main(url: str, snapshot_id: str):
"""Download media from a URL using yt-dlp."""
"""Download video/audio from a URL using yt-dlp."""
try:
# Check if media downloading is enabled
if not get_env_bool('MEDIA_ENABLED', True):
print('Skipping media (MEDIA_ENABLED=False)', file=sys.stderr)
# Check if yt-dlp downloading is enabled (YTDLP_ENABLED primary, MEDIA_ENABLED fallback)
ytdlp_enabled = get_env_bool('YTDLP_ENABLED', True) and get_env_bool('MEDIA_ENABLED', True)
if not ytdlp_enabled:
print('Skipping ytdlp (YTDLP_ENABLED=False)', file=sys.stderr)
# Temporary failure (config disabled) - NO JSONL emission
sys.exit(0)
# Check if staticfile extractor already handled this (permanent skip)
if has_staticfile_output():
print('Skipping media - staticfile extractor already downloaded this', file=sys.stderr)
print('Skipping ytdlp - staticfile extractor already downloaded this', file=sys.stderr)
print(json.dumps({'type': 'ArchiveResult', 'status': 'skipped', 'output_str': 'staticfile already exists'}))
sys.exit(0)
@@ -197,7 +210,7 @@ def main(url: str, snapshot_id: str):
binary = get_env('YTDLP_BINARY', 'yt-dlp')
# Run extraction
success, output, error = save_media(url, binary)
success, output, error = save_ytdlp(url, binary)
if success:
# Success - emit ArchiveResult

View File

@@ -1,5 +1,5 @@
<!-- Media embed - video/audio player -->
<div class="extractor-embed media-embed" style="width: 100%; height: 100%; min-height: 400px; background: #1a1a1a; display: flex; align-items: center; justify-content: center;">
<!-- YT-DLP embed - video/audio player -->
<div class="extractor-embed ytdlp-embed" style="width: 100%; height: 100%; min-height: 400px; background: #1a1a1a; display: flex; align-items: center; justify-content: center;">
<video src="{{ output_path }}"
style="max-width: 100%; max-height: 100%;"
controls

View File

@@ -1,5 +1,5 @@
<!-- Media fullscreen - full video/audio player -->
<div class="extractor-fullscreen media-fullscreen" style="width: 100%; height: 100vh; background: #000; display: flex; align-items: center; justify-content: center;">
<!-- YT-DLP fullscreen - full video/audio player -->
<div class="extractor-fullscreen ytdlp-fullscreen" style="width: 100%; height: 100vh; background: #000; display: flex; align-items: center; justify-content: center;">
<video src="{{ output_path }}"
style="max-width: 100%; max-height: 100%;"
controls

View File

@@ -1,5 +1,5 @@
<!-- Media thumbnail - shows video/audio player or placeholder -->
<div class="extractor-thumbnail media-thumbnail" style="width: 100%; height: 100px; overflow: hidden; background: #1a1a1a; display: flex; align-items: center; justify-content: center;">
<!-- YT-DLP thumbnail - shows video/audio player or placeholder -->
<div class="extractor-thumbnail ytdlp-thumbnail" style="width: 100%; height: 100px; overflow: hidden; background: #1a1a1a; display: flex; align-items: center; justify-content: center;">
<video src="{{ output_path }}"
style="width: 100%; height: 100px; object-fit: contain;"
poster=""
@@ -9,6 +9,6 @@
</video>
<div style="display: none; flex-direction: column; align-items: center; color: #888; font-size: 12px;">
<span style="font-size: 32px;">🎬</span>
<span>Media</span>
<span>YT-DLP</span>
</div>
</div>

View File

@@ -1,15 +1,14 @@
"""
Integration tests for media plugin
Integration tests for ytdlp plugin
Tests verify:
pass
1. Hook script exists
2. Dependencies installed via validation hooks
3. Verify deps with abx-pkg
4. Media extraction works on video URLs
4. YT-DLP extraction works on video URLs
5. JSONL output is correct
6. Config options work
7. Handles non-media URLs gracefully
6. Config options work (YTDLP_* and backwards-compatible MEDIA_* aliases)
7. Handles non-video URLs gracefully
"""
import json
@@ -22,12 +21,12 @@ import pytest
PLUGIN_DIR = Path(__file__).parent.parent
PLUGINS_ROOT = PLUGIN_DIR.parent
MEDIA_HOOK = next(PLUGIN_DIR.glob('on_Snapshot__*_media.*'), None)
YTDLP_HOOK = next(PLUGIN_DIR.glob('on_Snapshot__*_ytdlp.*'), None)
TEST_URL = 'https://example.com/video.mp4'
def test_hook_script_exists():
"""Verify on_Snapshot hook exists."""
assert MEDIA_HOOK.exists(), f"Hook not found: {MEDIA_HOOK}"
assert YTDLP_HOOK.exists(), f"Hook not found: {YTDLP_HOOK}"
def test_verify_deps_with_abx_pkg():
@@ -60,16 +59,16 @@ def test_verify_deps_with_abx_pkg():
if missing_binaries:
pass
def test_handles_non_media_url():
"""Test that media extractor handles non-media URLs gracefully via hook."""
def test_handles_non_video_url():
"""Test that ytdlp extractor handles non-video URLs gracefully via hook."""
# Prerequisites checked by earlier test
with tempfile.TemporaryDirectory() as tmpdir:
tmpdir = Path(tmpdir)
# Run media extraction hook on non-media URL
# Run ytdlp extraction hook on non-video URL
result = subprocess.run(
[sys.executable, str(MEDIA_HOOK), '--url', 'https://example.com', '--snapshot-id', 'test789'],
[sys.executable, str(YTDLP_HOOK), '--url', 'https://example.com', '--snapshot-id', 'test789'],
cwd=tmpdir,
capture_output=True,
text=True,
@@ -97,16 +96,16 @@ def test_handles_non_media_url():
assert result_json['status'] == 'succeeded', f"Should succeed: {result_json}"
def test_config_save_media_false_skips():
"""Test that MEDIA_ENABLED=False exits without emitting JSONL."""
def test_config_ytdlp_enabled_false_skips():
"""Test that YTDLP_ENABLED=False exits without emitting JSONL."""
import os
with tempfile.TemporaryDirectory() as tmpdir:
env = os.environ.copy()
env['MEDIA_ENABLED'] = 'False'
env['YTDLP_ENABLED'] = 'False'
result = subprocess.run(
[sys.executable, str(MEDIA_HOOK), '--url', TEST_URL, '--snapshot-id', 'test999'],
[sys.executable, str(YTDLP_HOOK), '--url', TEST_URL, '--snapshot-id', 'test999'],
cwd=tmpdir,
capture_output=True,
text=True,
@@ -124,17 +123,41 @@ def test_config_save_media_false_skips():
assert len(jsonl_lines) == 0, f"Should not emit JSONL when feature disabled, but got: {jsonl_lines}"
def test_config_timeout():
"""Test that MEDIA_TIMEOUT config is respected."""
def test_config_media_enabled_backwards_compat():
"""Test that MEDIA_ENABLED=False (backwards-compatible alias) also works."""
import os
with tempfile.TemporaryDirectory() as tmpdir:
env = os.environ.copy()
env['MEDIA_TIMEOUT'] = '5'
env['MEDIA_ENABLED'] = 'False'
result = subprocess.run(
[sys.executable, str(YTDLP_HOOK), '--url', TEST_URL, '--snapshot-id', 'test_compat'],
cwd=tmpdir,
capture_output=True,
text=True,
env=env,
timeout=30
)
assert result.returncode == 0, f"Should exit 0 when feature disabled via MEDIA_ENABLED: {result.stderr}"
# Should NOT emit any JSONL when disabled via backwards-compatible alias
jsonl_lines = [line for line in result.stdout.strip().split('\n') if line.strip().startswith('{')]
assert len(jsonl_lines) == 0, f"Should not emit JSONL when feature disabled via MEDIA_ENABLED, but got: {jsonl_lines}"
def test_config_timeout():
"""Test that YTDLP_TIMEOUT config is respected (also via MEDIA_TIMEOUT alias)."""
import os
with tempfile.TemporaryDirectory() as tmpdir:
env = os.environ.copy()
env['YTDLP_TIMEOUT'] = '5'
start_time = time.time()
result = subprocess.run(
[sys.executable, str(MEDIA_HOOK), '--url', 'https://example.com', '--snapshot-id', 'testtimeout'],
[sys.executable, str(YTDLP_HOOK), '--url', 'https://example.com', '--snapshot-id', 'testtimeout'],
cwd=tmpdir,
capture_output=True,
text=True,
@@ -149,7 +172,7 @@ def test_config_timeout():
def test_real_youtube_url():
"""Test that yt-dlp can extract media from a real YouTube URL."""
"""Test that yt-dlp can extract video/audio from a real YouTube URL."""
import os
with tempfile.TemporaryDirectory() as tmpdir:
@@ -159,11 +182,11 @@ def test_real_youtube_url():
youtube_url = 'https://www.youtube.com/watch?v=jNQXAC9IVRw' # "Me at the zoo" - first YouTube video
env = os.environ.copy()
env['MEDIA_TIMEOUT'] = '120' # Give it time to download
env['YTDLP_TIMEOUT'] = '120' # Give it time to download
start_time = time.time()
result = subprocess.run(
[sys.executable, str(MEDIA_HOOK), '--url', youtube_url, '--snapshot-id', 'testyoutube'],
[sys.executable, str(YTDLP_HOOK), '--url', youtube_url, '--snapshot-id', 'testyoutube'],
cwd=tmpdir,
capture_output=True,
text=True,
@@ -173,7 +196,7 @@ def test_real_youtube_url():
elapsed_time = time.time() - start_time
# Should succeed
assert result.returncode == 0, f"Should extract media successfully: {result.stderr}"
assert result.returncode == 0, f"Should extract video/audio successfully: {result.stderr}"
# Parse JSONL output
result_json = None
@@ -191,11 +214,11 @@ def test_real_youtube_url():
assert result_json, f"Should have ArchiveResult JSONL output. stdout: {result.stdout}"
assert result_json['status'] == 'succeeded', f"Should succeed: {result_json}"
# Check that some media files were downloaded
# Check that some video/audio files were downloaded
output_files = list(tmpdir.glob('**/*'))
media_files = [f for f in output_files if f.is_file() and f.suffix.lower() in ('.mp4', '.webm', '.mkv', '.m4a', '.mp3', '.json', '.jpg', '.webp')]
assert len(media_files) > 0, f"Should have downloaded at least one media file. Files: {output_files}"
assert len(media_files) > 0, f"Should have downloaded at least one video/audio file. Files: {output_files}"
print(f"Successfully extracted {len(media_files)} file(s) in {elapsed_time:.2f}s")

View File

@@ -458,7 +458,7 @@ class TestFilesystemMigration08to09(unittest.TestCase):
'SAVE_READABILITY': 'True',
'SAVE_MERCURY': 'True',
'SAVE_PDF': 'True',
'SAVE_MEDIA': 'True',
'SAVE_YTDLP': 'True',
'SAVE_ARCHIVEDOTORG': 'True',
'SAVE_HEADERS': 'True',
'SAVE_HTMLTOTEXT': 'True',

View File

@@ -1002,7 +1002,7 @@ def run_archivebox(data_dir: Path, args: list, timeout: int = 60, env: dict = No
base_env['SAVE_READABILITY'] = 'False'
base_env['SAVE_MERCURY'] = 'False'
base_env['SAVE_GIT'] = 'False'
base_env['SAVE_MEDIA'] = 'False'
base_env['SAVE_YTDLP'] = 'False'
base_env['SAVE_HEADERS'] = 'False'
base_env['SAVE_HTMLTOTEXT'] = 'False'

View File

@@ -32,7 +32,7 @@ services:
# - PUID=911 # set to your host user's UID & GID if you encounter permissions issues
# - PGID=911 # UID/GIDs lower than 500 may clash with system uids and are not recommended
# For options below, it's better to set in data/ArchiveBox.conf or use `docker compose run archivebox config --set SOME_KEY=someval` instead of setting here:
# - MEDIA_MAX_SIZE=750m # increase this filesize limit to allow archiving larger audio/video files
# - YTDLP_MAX_SIZE=750m # increase this filesize limit to allow archiving larger video/audio files
# - TIMEOUT=60 # increase this number to 120+ seconds if you see many slow downloads timing out
# - CHECK_SSL_VALIDITY=True # set to False to disable strict SSL checking (allows saving URLs w/ broken certs)
# - SAVE_ARCHIVEDOTORG=True # set to False to disable submitting all URLs to Archive.org when archiving

View File

@@ -11,7 +11,7 @@
# RESTRICT_FILE_NAMES = windows
# ONLY_NEW = False
# TIMEOUT = 60
# MEDIA_TIMEOUT = 3600
# YTDLP_TIMEOUT = 3600
# URL_BLACKLIST = (://(.*\.)?facebook\.com)|(://(.*\.)?ebay\.com)|(.*\.exe$)
# CHECK_SSL_VALIDITY = True
# RESOLUTION = 1440,900
@@ -36,7 +36,7 @@
# SAVE_SCREENSHOT = True
# SAVE_DOM = True
# SAVE_GIT = True
# SAVE_MEDIA = False
# SAVE_YTDLP = False
# SAVE_ARCHIVE_DOT_ORG = True

View File

@@ -23,7 +23,7 @@ def disable_extractors_dict():
"SAVE_DOM": "false",
"SAVE_HEADERS": "false",
"USE_GIT": "false",
"SAVE_MEDIA": "false",
"SAVE_YTDLP": "false",
"SAVE_ARCHIVEDOTORG": "false",
"SAVE_TITLE": "false",
"SAVE_FAVICON": "false",

View File

@@ -83,7 +83,7 @@ def test_config_set_multiple_values(tmp_path, process):
os.chdir(tmp_path)
result = subprocess.run(
['archivebox', 'config', '--set', 'TIMEOUT=111', 'MEDIA_TIMEOUT=222'],
['archivebox', 'config', '--set', 'TIMEOUT=111', 'YTDLP_TIMEOUT=222'],
capture_output=True,
text=True,
)
@@ -149,7 +149,7 @@ def test_config_preserves_existing_values(tmp_path, process):
# Set second value
subprocess.run(
['archivebox', 'config', '--set', 'MEDIA_TIMEOUT=200'],
['archivebox', 'config', '--set', 'YTDLP_TIMEOUT=200'],
capture_output=True,
)
@@ -157,7 +157,7 @@ def test_config_preserves_existing_values(tmp_path, process):
config_file = tmp_path / 'ArchiveBox.conf'
content = config_file.read_text()
assert 'TIMEOUT' in content
assert 'MEDIA_TIMEOUT' in content
assert 'YTDLP_TIMEOUT' in content
def test_config_file_is_valid_toml(tmp_path, process):

View File

@@ -32,7 +32,7 @@ def test_background_hooks_dont_block_parser_extractors(tmp_path, process):
"SAVE_DOM": "false",
"SAVE_HEADERS": "false",
"USE_GIT": "false",
"SAVE_MEDIA": "false",
"SAVE_YTDLP": "false",
"SAVE_ARCHIVEDOTORG": "false",
"SAVE_TITLE": "false",
"SAVE_FAVICON": "false",
@@ -132,7 +132,7 @@ def test_parser_extractors_emit_snapshot_jsonl(tmp_path, process):
"SAVE_DOM": "false",
"SAVE_HEADERS": "false",
"USE_GIT": "false",
"SAVE_MEDIA": "false",
"SAVE_YTDLP": "false",
"SAVE_ARCHIVEDOTORG": "false",
"SAVE_TITLE": "false",
"SAVE_FAVICON": "false",