mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-04-05 15:27:53 +10:00
new gallerydl plugin and more
This commit is contained in:
Nick Sweeting
45
archivebox/plugins/gallerydl/config.json
Normal file
45
archivebox/plugins/gallerydl/config.json
Normal file
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"SAVE_GALLERY_DL": {
|
||||
"type": "boolean",
|
||||
"default": true,
|
||||
"x-aliases": ["USE_GALLERY_DL", "FETCH_GALLERY"],
|
||||
"description": "Enable gallery downloading with gallery-dl"
|
||||
},
|
||||
"GALLERY_DL_BINARY": {
|
||||
"type": "string",
|
||||
"default": "gallery-dl",
|
||||
"description": "Path to gallery-dl binary"
|
||||
},
|
||||
"GALLERY_DL_TIMEOUT": {
|
||||
"type": "integer",
|
||||
"default": 3600,
|
||||
"minimum": 30,
|
||||
"x-fallback": "TIMEOUT",
|
||||
"description": "Timeout for gallery downloads in seconds"
|
||||
},
|
||||
"GALLERY_DL_CHECK_SSL_VALIDITY": {
|
||||
"type": "boolean",
|
||||
"default": true,
|
||||
"x-fallback": "CHECK_SSL_VALIDITY",
|
||||
"description": "Whether to verify SSL certificates"
|
||||
},
|
||||
"GALLERY_DL_ARGS": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"default": [
|
||||
"--write-metadata",
|
||||
"--write-info-json"
|
||||
],
|
||||
"description": "Default gallery-dl arguments"
|
||||
},
|
||||
"GALLERY_DL_EXTRA_ARGS": {
|
||||
"type": "string",
|
||||
"default": "",
|
||||
"description": "Extra arguments for gallery-dl (space-separated)"
|
||||
}
|
||||
}
|
||||
}
|
||||
129
archivebox/plugins/gallerydl/on_Crawl__00_validate_gallerydl.py
Executable file
129
archivebox/plugins/gallerydl/on_Crawl__00_validate_gallerydl.py
Executable file
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Validation hook for gallery-dl.
|
||||
|
||||
Runs at crawl start to verify gallery-dl binary is available.
|
||||
Outputs JSONL for InstalledBinary and Machine config updates.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import shutil
|
||||
import hashlib
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def get_binary_version(abspath: str, version_flag: str = '--version') -> str | None:
|
||||
"""Get version string from binary."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[abspath, version_flag],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=5,
|
||||
)
|
||||
if result.returncode == 0 and result.stdout:
|
||||
first_line = result.stdout.strip().split('\n')[0]
|
||||
return first_line[:64]
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def get_binary_hash(abspath: str) -> str | None:
|
||||
"""Get SHA256 hash of binary."""
|
||||
try:
|
||||
with open(abspath, 'rb') as f:
|
||||
return hashlib.sha256(f.read()).hexdigest()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def find_gallerydl() -> dict | None:
|
||||
"""Find gallery-dl binary."""
|
||||
try:
|
||||
from abx_pkg import Binary, PipProvider, EnvProvider
|
||||
|
||||
class GalleryDlBinary(Binary):
|
||||
name: str = 'gallery-dl'
|
||||
binproviders_supported = [PipProvider(), EnvProvider()]
|
||||
|
||||
binary = GalleryDlBinary()
|
||||
loaded = binary.load()
|
||||
if loaded and loaded.abspath:
|
||||
return {
|
||||
'name': 'gallery-dl',
|
||||
'abspath': str(loaded.abspath),
|
||||
'version': str(loaded.version) if loaded.version else None,
|
||||
'sha256': loaded.sha256 if hasattr(loaded, 'sha256') else None,
|
||||
'binprovider': loaded.binprovider.name if loaded.binprovider else 'env',
|
||||
}
|
||||
except ImportError:
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback to shutil.which
|
||||
abspath = shutil.which('gallery-dl') or os.environ.get('GALLERY_DL_BINARY', '')
|
||||
if abspath and Path(abspath).is_file():
|
||||
return {
|
||||
'name': 'gallery-dl',
|
||||
'abspath': abspath,
|
||||
'version': get_binary_version(abspath),
|
||||
'sha256': get_binary_hash(abspath),
|
||||
'binprovider': 'env',
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
# Check for gallery-dl (required)
|
||||
gallerydl_result = find_gallerydl()
|
||||
|
||||
missing_deps = []
|
||||
|
||||
# Emit results for gallery-dl
|
||||
if gallerydl_result and gallerydl_result.get('abspath'):
|
||||
print(json.dumps({
|
||||
'type': 'InstalledBinary',
|
||||
'name': gallerydl_result['name'],
|
||||
'abspath': gallerydl_result['abspath'],
|
||||
'version': gallerydl_result['version'],
|
||||
'sha256': gallerydl_result['sha256'],
|
||||
'binprovider': gallerydl_result['binprovider'],
|
||||
}))
|
||||
|
||||
print(json.dumps({
|
||||
'type': 'Machine',
|
||||
'_method': 'update',
|
||||
'key': 'config/GALLERY_DL_BINARY',
|
||||
'value': gallerydl_result['abspath'],
|
||||
}))
|
||||
|
||||
if gallerydl_result['version']:
|
||||
print(json.dumps({
|
||||
'type': 'Machine',
|
||||
'_method': 'update',
|
||||
'key': 'config/GALLERY_DL_VERSION',
|
||||
'value': gallerydl_result['version'],
|
||||
}))
|
||||
else:
|
||||
print(json.dumps({
|
||||
'type': 'Dependency',
|
||||
'bin_name': 'gallery-dl',
|
||||
'bin_providers': 'pip,env',
|
||||
}))
|
||||
missing_deps.append('gallery-dl')
|
||||
|
||||
if missing_deps:
|
||||
print(f"Missing dependencies: {', '.join(missing_deps)}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
else:
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
299
archivebox/plugins/gallerydl/on_Snapshot__52_gallerydl.py
Executable file
299
archivebox/plugins/gallerydl/on_Snapshot__52_gallerydl.py
Executable file
@@ -0,0 +1,299 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Download image galleries from a URL using gallery-dl.
|
||||
|
||||
Usage: on_Snapshot__gallerydl.py --url=<url> --snapshot-id=<uuid>
|
||||
Output: Downloads gallery images to $PWD/gallerydl/
|
||||
|
||||
Environment variables:
|
||||
GALLERY_DL_BINARY: Path to gallery-dl binary
|
||||
GALLERY_DL_TIMEOUT: Timeout in seconds (default: 3600 for large galleries)
|
||||
GALLERY_DL_CHECK_SSL_VALIDITY: Whether to check SSL certificates (default: True)
|
||||
GALLERY_DL_EXTRA_ARGS: Extra arguments for gallery-dl (space-separated)
|
||||
|
||||
# Gallery-dl feature toggles
|
||||
USE_GALLERY_DL: Enable gallery-dl gallery extraction (default: True)
|
||||
SAVE_GALLERY_DL: Alias for USE_GALLERY_DL
|
||||
|
||||
# Fallback to ARCHIVING_CONFIG values if GALLERY_DL_* not set:
|
||||
GALLERY_DL_TIMEOUT: Fallback timeout for gallery downloads
|
||||
TIMEOUT: Fallback timeout
|
||||
CHECK_SSL_VALIDITY: Fallback SSL check
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import rich_click as click
|
||||
|
||||
|
||||
# Extractor metadata
|
||||
EXTRACTOR_NAME = 'gallerydl'
|
||||
BIN_NAME = 'gallery-dl'
|
||||
BIN_PROVIDERS = 'pip,env'
|
||||
OUTPUT_DIR = '.'
|
||||
|
||||
|
||||
def get_env(name: str, default: str = '') -> str:
|
||||
return os.environ.get(name, default).strip()
|
||||
|
||||
|
||||
def get_env_bool(name: str, default: bool = False) -> bool:
|
||||
val = get_env(name, '').lower()
|
||||
if val in ('true', '1', 'yes', 'on'):
|
||||
return True
|
||||
if val in ('false', '0', 'no', 'off'):
|
||||
return False
|
||||
return default
|
||||
|
||||
|
||||
def get_env_int(name: str, default: int = 0) -> int:
|
||||
try:
|
||||
return int(get_env(name, str(default)))
|
||||
except ValueError:
|
||||
return default
|
||||
|
||||
|
||||
STATICFILE_DIR = '../staticfile'
|
||||
MEDIA_DIR = '../media'
|
||||
|
||||
def has_staticfile_output() -> bool:
|
||||
"""Check if staticfile extractor already downloaded this URL."""
|
||||
staticfile_dir = Path(STATICFILE_DIR)
|
||||
return staticfile_dir.exists() and any(staticfile_dir.iterdir())
|
||||
|
||||
|
||||
def has_media_output() -> bool:
|
||||
"""Check if media extractor already downloaded this URL."""
|
||||
media_dir = Path(MEDIA_DIR)
|
||||
return media_dir.exists() and any(media_dir.iterdir())
|
||||
|
||||
|
||||
def find_gallerydl() -> str | None:
|
||||
"""Find gallery-dl binary."""
|
||||
gallerydl = get_env('GALLERY_DL_BINARY')
|
||||
if gallerydl and os.path.isfile(gallerydl):
|
||||
return gallerydl
|
||||
|
||||
binary = shutil.which('gallery-dl')
|
||||
if binary:
|
||||
return binary
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_version(binary: str) -> str:
|
||||
"""Get gallery-dl version."""
|
||||
try:
|
||||
result = subprocess.run([binary, '--version'], capture_output=True, text=True, timeout=10)
|
||||
return result.stdout.strip()[:64]
|
||||
except Exception:
|
||||
return ''
|
||||
|
||||
|
||||
# Default gallery-dl args
|
||||
def get_gallerydl_default_args() -> list[str]:
|
||||
"""Build default gallery-dl arguments."""
|
||||
return [
|
||||
'--write-metadata',
|
||||
'--write-info-json',
|
||||
]
|
||||
|
||||
|
||||
def save_gallery(url: str, binary: str) -> tuple[bool, str | None, str]:
|
||||
"""
|
||||
Download gallery using gallery-dl.
|
||||
|
||||
Returns: (success, output_path, error_message)
|
||||
"""
|
||||
# Get config from env (with GALLERY_DL_ prefix or fallback to ARCHIVING_CONFIG style)
|
||||
timeout = get_env_int('GALLERY_DL_TIMEOUT') or get_env_int('TIMEOUT', 3600)
|
||||
check_ssl = get_env_bool('GALLERY_DL_CHECK_SSL_VALIDITY', get_env_bool('CHECK_SSL_VALIDITY', True))
|
||||
extra_args = get_env('GALLERY_DL_EXTRA_ARGS', '')
|
||||
|
||||
# Output directory is current directory (hook already runs in output dir)
|
||||
output_dir = Path(OUTPUT_DIR)
|
||||
|
||||
# Build command (later options take precedence)
|
||||
cmd = [
|
||||
binary,
|
||||
*get_gallerydl_default_args(),
|
||||
'-d', str(output_dir),
|
||||
]
|
||||
|
||||
if not check_ssl:
|
||||
cmd.append('--no-check-certificate')
|
||||
|
||||
if extra_args:
|
||||
cmd.extend(extra_args.split())
|
||||
|
||||
cmd.append(url)
|
||||
|
||||
try:
|
||||
result = subprocess.run(cmd, capture_output=True, timeout=timeout, text=True)
|
||||
|
||||
# Check if any gallery files were downloaded
|
||||
gallery_extensions = (
|
||||
'.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.svg',
|
||||
'.mp4', '.webm', '.mkv', '.avi', '.mov', '.flv',
|
||||
'.json', '.txt', '.zip',
|
||||
)
|
||||
|
||||
downloaded_files = [
|
||||
f for f in output_dir.glob('*')
|
||||
if f.is_file() and f.suffix.lower() in gallery_extensions
|
||||
]
|
||||
|
||||
if downloaded_files:
|
||||
# Return first image file, or first file if no images
|
||||
image_files = [
|
||||
f for f in downloaded_files
|
||||
if f.suffix.lower() in ('.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp')
|
||||
]
|
||||
output = str(image_files[0]) if image_files else str(downloaded_files[0])
|
||||
return True, output, ''
|
||||
else:
|
||||
stderr = result.stderr
|
||||
|
||||
# These are NOT errors - page simply has no downloadable gallery
|
||||
# Return success with no output (legitimate "nothing to download")
|
||||
if 'unsupported URL' in stderr.lower():
|
||||
return True, None, '' # Not a gallery site - success, no output
|
||||
if 'no results' in stderr.lower():
|
||||
return True, None, '' # No gallery found - success, no output
|
||||
if result.returncode == 0:
|
||||
return True, None, '' # gallery-dl exited cleanly, just no gallery - success
|
||||
|
||||
# These ARE errors - something went wrong
|
||||
if '404' in stderr:
|
||||
return False, None, '404 Not Found'
|
||||
if '403' in stderr:
|
||||
return False, None, '403 Forbidden'
|
||||
if 'Unable to extract' in stderr:
|
||||
return False, None, 'Unable to extract gallery info'
|
||||
|
||||
return False, None, f'gallery-dl error: {stderr[:200]}'
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
return False, None, f'Timed out after {timeout} seconds'
|
||||
except Exception as e:
|
||||
return False, None, f'{type(e).__name__}: {e}'
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option('--url', required=True, help='URL to download gallery from')
|
||||
@click.option('--snapshot-id', required=True, help='Snapshot UUID')
|
||||
def main(url: str, snapshot_id: str):
|
||||
"""Download image gallery from a URL using gallery-dl."""
|
||||
|
||||
start_ts = datetime.now(timezone.utc)
|
||||
version = ''
|
||||
output = None
|
||||
status = 'failed'
|
||||
error = ''
|
||||
binary = None
|
||||
cmd_str = ''
|
||||
|
||||
try:
|
||||
# Check if gallery-dl is enabled
|
||||
if not (get_env_bool('USE_GALLERY_DL', True) and get_env_bool('SAVE_GALLERY_DL', True)):
|
||||
print('Skipping gallery-dl (USE_GALLERY_DL=False or SAVE_GALLERY_DL=False)')
|
||||
status = 'skipped'
|
||||
end_ts = datetime.now(timezone.utc)
|
||||
print(f'START_TS={start_ts.isoformat()}')
|
||||
print(f'END_TS={end_ts.isoformat()}')
|
||||
print(f'STATUS={status}')
|
||||
print(f'RESULT_JSON={json.dumps({"extractor": EXTRACTOR_NAME, "status": status, "url": url, "snapshot_id": snapshot_id})}')
|
||||
sys.exit(0)
|
||||
|
||||
# Check if staticfile or media extractors already handled this (skip)
|
||||
if has_staticfile_output():
|
||||
print(f'Skipping gallery-dl - staticfile extractor already downloaded this')
|
||||
status = 'skipped'
|
||||
print(f'START_TS={start_ts.isoformat()}')
|
||||
print(f'END_TS={datetime.now(timezone.utc).isoformat()}')
|
||||
print(f'STATUS={status}')
|
||||
print(f'RESULT_JSON={json.dumps({"extractor": EXTRACTOR_NAME, "status": status, "url": url, "snapshot_id": snapshot_id})}')
|
||||
sys.exit(0)
|
||||
|
||||
if has_media_output():
|
||||
print(f'Skipping gallery-dl - media extractor already downloaded this')
|
||||
status = 'skipped'
|
||||
print(f'START_TS={start_ts.isoformat()}')
|
||||
print(f'END_TS={datetime.now(timezone.utc).isoformat()}')
|
||||
print(f'STATUS={status}')
|
||||
print(f'RESULT_JSON={json.dumps({"extractor": EXTRACTOR_NAME, "status": status, "url": url, "snapshot_id": snapshot_id})}')
|
||||
sys.exit(0)
|
||||
|
||||
# Find binary
|
||||
binary = find_gallerydl()
|
||||
if not binary:
|
||||
print(f'ERROR: {BIN_NAME} binary not found', file=sys.stderr)
|
||||
print(f'DEPENDENCY_NEEDED={BIN_NAME}', file=sys.stderr)
|
||||
print(f'BIN_PROVIDERS={BIN_PROVIDERS}', file=sys.stderr)
|
||||
print(f'INSTALL_HINT=pip install gallery-dl', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
version = get_version(binary)
|
||||
cmd_str = f'{binary} {url}'
|
||||
|
||||
# Run extraction
|
||||
success, output, error = save_gallery(url, binary)
|
||||
status = 'succeeded' if success else 'failed'
|
||||
|
||||
if success:
|
||||
output_dir = Path(OUTPUT_DIR)
|
||||
files = list(output_dir.glob('*'))
|
||||
file_count = len([f for f in files if f.is_file()])
|
||||
if file_count > 0:
|
||||
print(f'gallery-dl completed: {file_count} files downloaded')
|
||||
else:
|
||||
print(f'gallery-dl completed: no gallery found on page (this is normal)')
|
||||
|
||||
except Exception as e:
|
||||
error = f'{type(e).__name__}: {e}'
|
||||
status = 'failed'
|
||||
|
||||
# Print results
|
||||
end_ts = datetime.now(timezone.utc)
|
||||
duration = (end_ts - start_ts).total_seconds()
|
||||
|
||||
print(f'START_TS={start_ts.isoformat()}')
|
||||
print(f'END_TS={end_ts.isoformat()}')
|
||||
print(f'DURATION={duration:.2f}')
|
||||
if cmd_str:
|
||||
print(f'CMD={cmd_str}')
|
||||
if version:
|
||||
print(f'VERSION={version}')
|
||||
if output:
|
||||
print(f'OUTPUT={output}')
|
||||
print(f'STATUS={status}')
|
||||
|
||||
if error:
|
||||
print(f'ERROR={error}', file=sys.stderr)
|
||||
|
||||
# Print JSON result
|
||||
result_json = {
|
||||
'extractor': EXTRACTOR_NAME,
|
||||
'url': url,
|
||||
'snapshot_id': snapshot_id,
|
||||
'status': status,
|
||||
'start_ts': start_ts.isoformat(),
|
||||
'end_ts': end_ts.isoformat(),
|
||||
'duration': round(duration, 2),
|
||||
'cmd_version': version,
|
||||
'output': output,
|
||||
'error': error or None,
|
||||
}
|
||||
print(f'RESULT_JSON={json.dumps(result_json)}')
|
||||
|
||||
sys.exit(0 if status == 'succeeded' else 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user