add overrides options to binproviders

archivebox/plugins/chrome_session/on_Crawl__00_validate_chrome.py

@@ -6,103 +6,29 @@ Runs at crawl start to verify Chrome is available.
 Outputs JSONL for InstalledBinary and Machine config updates.
 """
 
-import os
 import sys
 import json
-import shutil
-import hashlib
-import subprocess
-from pathlib import Path
-
-
-# Common Chrome/Chromium binary names and paths
-CHROME_NAMES = [
-    'chromium',
-    'chromium-browser',
-    'google-chrome',
-    'google-chrome-stable',
-    'chrome',
-]
-
-CHROME_PATHS = [
-    '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
-    '/Applications/Chromium.app/Contents/MacOS/Chromium',
-    '/usr/bin/google-chrome',
-    '/usr/bin/google-chrome-stable',
-    '/usr/bin/chromium',
-    '/usr/bin/chromium-browser',
-    '/snap/bin/chromium',
-    '/opt/google/chrome/chrome',
-]
-
-
-def get_binary_version(abspath: str) -> str | None:
-    """Get version string from Chrome binary."""
-    try:
-        result = subprocess.run(
-            [abspath, '--version'],
-            capture_output=True,
-            text=True,
-            timeout=10,
-        )
-        if result.returncode == 0 and result.stdout:
-            # Chrome version string: "Google Chrome 120.0.6099.109" or "Chromium 120.0.6099.109"
-            first_line = result.stdout.strip().split('\n')[0]
-            parts = first_line.split()
-            # Find version number (looks like 120.0.6099.109)
-            for part in parts:
-                if '.' in part and part[0].isdigit():
-                    return part
-            return first_line[:32]
-    except Exception:
-        pass
-    return None
-
-
-def get_binary_hash(abspath: str) -> str | None:
-    """Get SHA256 hash of binary."""
-    try:
-        with open(abspath, 'rb') as f:
-            return hashlib.sha256(f.read()).hexdigest()
-    except Exception:
-        return None
 
 
 def find_chrome() -> dict | None:
     """Find Chrome/Chromium binary."""
-    # Check env var first
-    env_path = os.environ.get('CHROME_BINARY', '')
-    if env_path and Path(env_path).is_file():
-        return {
-            'name': 'chrome',
-            'abspath': env_path,
-            'version': get_binary_version(env_path),
-            'sha256': get_binary_hash(env_path),
-            'binprovider': 'env',
-        }
+    try:
+        from abx_pkg import Binary, AptProvider, BrewProvider, EnvProvider
 
-    # Try shutil.which for various names
-    for name in CHROME_NAMES:
-        abspath = shutil.which(name)
-        if abspath:
-            return {
-                'name': 'chrome',
-                'abspath': abspath,
-                'version': get_binary_version(abspath),
-                'sha256': get_binary_hash(abspath),
-                'binprovider': 'env',
-            }
-
-    # Check common paths
-    for path in CHROME_PATHS:
-        if Path(path).is_file():
-            return {
-                'name': 'chrome',
-                'abspath': path,
-                'version': get_binary_version(path),
-                'sha256': get_binary_hash(path),
-                'binprovider': 'env',
-            }
+        # Try common Chrome/Chromium binary names
+        for name in ['google-chrome', 'chromium', 'chromium-browser', 'google-chrome-stable', 'chrome']:
+            binary = Binary(name=name, binproviders=[AptProvider(), BrewProvider(), EnvProvider()])
+            loaded = binary.load()
+            if loaded and loaded.abspath:
+                return {
+                    'name': 'chrome',
+                    'abspath': str(loaded.abspath),
+                    'version': str(loaded.version) if loaded.version else None,
+                    'sha256': loaded.sha256 if hasattr(loaded, 'sha256') else None,
+                    'binprovider': loaded.binprovider.name if loaded.binprovider else 'env',
+                }
+    except Exception:
+        pass
 
     return None
 

archivebox/plugins/forumdl/on_Crawl__00_validate_forumdl.py

@@ -6,13 +6,8 @@ Runs at crawl start to verify forum-dl binary is available.
 Outputs JSONL for InstalledBinary and Machine config updates.
 """
 
-import os
 import sys
 import json
-import shutil
-import hashlib
-import subprocess
-from pathlib import Path
 
 
 def find_forumdl() -> dict | None:
@@ -30,22 +25,9 @@ def find_forumdl() -> dict | None:
                 'sha256': loaded.sha256 if hasattr(loaded, 'sha256') else None,
                 'binprovider': loaded.binprovider.name if loaded.binprovider else 'env',
             }
-    except ImportError:
-        pass
     except Exception:
         pass
 
-    # Fallback to shutil.which
-    abspath = shutil.which('forum-dl') or os.environ.get('FORUMDL_BINARY', '')
-    if abspath and Path(abspath).is_file():
-        return {
-            'name': 'forum-dl',
-            'abspath': abspath,
-            'version': get_binary_version(abspath),
-            'sha256': get_binary_hash(abspath),
-            'binprovider': 'env',
-        }
-
     return None
 
 

archivebox/plugins/gallerydl/on_Crawl__00_validate_gallerydl.py

@@ -6,39 +6,8 @@ Runs at crawl start to verify gallery-dl binary is available.
 Outputs JSONL for InstalledBinary and Machine config updates.
 """
 
-import os
 import sys
 import json
-import shutil
-import hashlib
-import subprocess
-from pathlib import Path
-
-
-def get_binary_version(abspath: str, version_flag: str = '--version') -> str | None:
-    """Get version string from binary."""
-    try:
-        result = subprocess.run(
-            [abspath, version_flag],
-            capture_output=True,
-            text=True,
-            timeout=5,
-        )
-        if result.returncode == 0 and result.stdout:
-            first_line = result.stdout.strip().split('\n')[0]
-            return first_line[:64]
-    except Exception:
-        pass
-    return None
-
-
-def get_binary_hash(abspath: str) -> str | None:
-    """Get SHA256 hash of binary."""
-    try:
-        with open(abspath, 'rb') as f:
-            return hashlib.sha256(f.read()).hexdigest()
-    except Exception:
-        return None
 
 
 def find_gallerydl() -> dict | None:
@@ -46,11 +15,7 @@ def find_gallerydl() -> dict | None:
     try:
         from abx_pkg import Binary, PipProvider, EnvProvider
 
-        class GalleryDlBinary(Binary):
-            name: str = 'gallery-dl'
-            binproviders_supported = [PipProvider(), EnvProvider()]
-
-        binary = GalleryDlBinary()
+        binary = Binary(name='gallery-dl', binproviders=[PipProvider(), EnvProvider()])
         loaded = binary.load()
         if loaded and loaded.abspath:
             return {
@@ -60,22 +25,9 @@ def find_gallerydl() -> dict | None:
                 'sha256': loaded.sha256 if hasattr(loaded, 'sha256') else None,
                 'binprovider': loaded.binprovider.name if loaded.binprovider else 'env',
             }
-    except ImportError:
-        pass
     except Exception:
         pass
 
-    # Fallback to shutil.which
-    abspath = shutil.which('gallery-dl') or os.environ.get('GALLERYDL_BINARY', '')
-    if abspath and Path(abspath).is_file():
-        return {
-            'name': 'gallery-dl',
-            'abspath': abspath,
-            'version': get_binary_version(abspath),
-            'sha256': get_binary_hash(abspath),
-            'binprovider': 'env',
-        }
-
     return None
 
 

archivebox/plugins/git/on_Crawl__00_validate_git.py

@@ -6,43 +6,8 @@ Runs at crawl start to verify git is available.
 Outputs JSONL for InstalledBinary and Machine config updates.
 """
 
-import os
 import sys
 import json
-import shutil
-import hashlib
-import subprocess
-from pathlib import Path
-
-
-def get_binary_version(abspath: str) -> str | None:
-    """Get version string from binary."""
-    try:
-        result = subprocess.run(
-            [abspath, '--version'],
-            capture_output=True,
-            text=True,
-            timeout=5,
-        )
-        if result.returncode == 0 and result.stdout:
-            # git version string: "git version 2.43.0"
-            first_line = result.stdout.strip().split('\n')[0]
-            parts = first_line.split()
-            if len(parts) >= 3 and parts[0] == 'git':
-                return parts[2]
-            return first_line[:32]
-    except Exception:
-        pass
-    return None
-
-
-def get_binary_hash(abspath: str) -> str | None:
-    """Get SHA256 hash of binary."""
-    try:
-        with open(abspath, 'rb') as f:
-            return hashlib.sha256(f.read()).hexdigest()
-    except Exception:
-        return None
 
 
 def find_git() -> dict | None:
@@ -50,11 +15,7 @@ def find_git() -> dict | None:
     try:
         from abx_pkg import Binary, EnvProvider
 
-        class GitBinary(Binary):
-            name: str = 'git'
-            binproviders_supported = [EnvProvider()]
-
-        binary = GitBinary()
+        binary = Binary(name='git', binproviders=[EnvProvider()])
         loaded = binary.load()
         if loaded and loaded.abspath:
             return {
@@ -64,22 +25,9 @@ def find_git() -> dict | None:
                 'sha256': loaded.sha256 if hasattr(loaded, 'sha256') else None,
                 'binprovider': loaded.binprovider.name if loaded.binprovider else 'env',
             }
-    except ImportError:
-        pass
     except Exception:
         pass
 
-    # Fallback to shutil.which
-    abspath = shutil.which('git') or os.environ.get('GIT_BINARY', '')
-    if abspath and Path(abspath).is_file():
-        return {
-            'name': 'git',
-            'abspath': abspath,
-            'version': get_binary_version(abspath),
-            'sha256': get_binary_hash(abspath),
-            'binprovider': 'env',
-        }
-
     return None
 
 

archivebox/plugins/media/on_Crawl__00_validate_ytdlp.py

@@ -6,13 +6,8 @@ Runs at crawl start to verify yt-dlp and required binaries are available.
 Outputs JSONL for InstalledBinary and Machine config updates.
 """
 
-import os
 import sys
 import json
-import shutil
-import hashlib
-import subprocess
-from pathlib import Path
 
 
 def find_ytdlp() -> dict | None:
@@ -30,22 +25,9 @@ def find_ytdlp() -> dict | None:
                 'sha256': loaded.sha256 if hasattr(loaded, 'sha256') else None,
                 'binprovider': loaded.binprovider.name if loaded.binprovider else 'env',
             }
-    except ImportError:
-        pass
     except Exception:
         pass
 
-    # Fallback to shutil.which
-    abspath = shutil.which('yt-dlp') or os.environ.get('YTDLP_BINARY', '')
-    if abspath and Path(abspath).is_file():
-        return {
-            'name': 'yt-dlp',
-            'abspath': abspath,
-            'version': None,
-            'sha256': None,
-            'binprovider': 'env',
-        }
-
     return None
 
 
@@ -64,22 +46,9 @@ def find_node() -> dict | None:
                 'sha256': loaded.sha256 if hasattr(loaded, 'sha256') else None,
                 'binprovider': loaded.binprovider.name if loaded.binprovider else 'env',
             }
-    except ImportError:
-        pass
     except Exception:
         pass
 
-    # Fallback to shutil.which
-    abspath = shutil.which('node') or os.environ.get('NODE_BINARY', '')
-    if abspath and Path(abspath).is_file():
-        return {
-            'name': 'node',
-            'abspath': abspath,
-            'version': None,
-            'sha256': None,
-            'binprovider': 'env',
-        }
-
     return None
 
 
@@ -98,22 +67,9 @@ def find_ffmpeg() -> dict | None:
                 'sha256': loaded.sha256 if hasattr(loaded, 'sha256') else None,
                 'binprovider': loaded.binprovider.name if loaded.binprovider else 'env',
             }
-    except ImportError:
-        pass
     except Exception:
         pass
 
-    # Fallback to shutil.which
-    abspath = shutil.which('ffmpeg') or os.environ.get('FFMPEG_BINARY', '')
-    if abspath and Path(abspath).is_file():
-        return {
-            'name': 'ffmpeg',
-            'abspath': abspath,
-            'version': None,
-            'sha256': None,
-            'binprovider': 'env',
-        }
-
     return None
 
 

archivebox/plugins/mercury/on_Crawl__00_validate_mercury.py

@@ -6,13 +6,8 @@ Runs at crawl start to verify postlight-parser is available.
 Outputs JSONL for InstalledBinary and Machine config updates.
 """
 
-import os
 import sys
 import json
-import shutil
-import hashlib
-import subprocess
-from pathlib import Path
 
 
 def find_mercury() -> dict | None:
@@ -30,22 +25,9 @@ def find_mercury() -> dict | None:
                 'sha256': loaded.sha256 if hasattr(loaded, 'sha256') else None,
                 'binprovider': loaded.binprovider.name if loaded.binprovider else 'env',
             }
-    except ImportError:
-        pass
     except Exception:
         pass
 
-    # Fallback to shutil.which
-    abspath = shutil.which('postlight-parser') or os.environ.get('MERCURY_BINARY', '')
-    if abspath and Path(abspath).is_file():
-        return {
-            'name': 'postlight-parser',
-            'abspath': abspath,
-            'version': None,
-            'sha256': None,
-            'binprovider': 'env',
-        }
-
     return None
 
 

archivebox/plugins/papersdl/on_Crawl__00_validate_papersdl.py

@@ -6,39 +6,8 @@ Runs at crawl start to verify papers-dl binary is available.
 Outputs JSONL for InstalledBinary and Machine config updates.
 """
 
-import os
 import sys
 import json
-import shutil
-import hashlib
-import subprocess
-from pathlib import Path
-
-
-def get_binary_version(abspath: str, version_flag: str = '--version') -> str | None:
-    """Get version string from binary."""
-    try:
-        result = subprocess.run(
-            [abspath, version_flag],
-            capture_output=True,
-            text=True,
-            timeout=5,
-        )
-        if result.returncode == 0 and result.stdout:
-            first_line = result.stdout.strip().split('\n')[0]
-            return first_line[:64]
-    except Exception:
-        pass
-    return None
-
-
-def get_binary_hash(abspath: str) -> str | None:
-    """Get SHA256 hash of binary."""
-    try:
-        with open(abspath, 'rb') as f:
-            return hashlib.sha256(f.read()).hexdigest()
-    except Exception:
-        return None
 
 
 def find_papersdl() -> dict | None:
@@ -46,11 +15,7 @@ def find_papersdl() -> dict | None:
     try:
         from abx_pkg import Binary, PipProvider, EnvProvider
 
-        class PapersdlBinary(Binary):
-            name: str = 'papers-dl'
-            binproviders_supported = [PipProvider(), EnvProvider()]
-
-        binary = PapersdlBinary()
+        binary = Binary(name='papers-dl', binproviders=[PipProvider(), EnvProvider()])
         loaded = binary.load()
         if loaded and loaded.abspath:
             return {
@@ -60,22 +25,9 @@ def find_papersdl() -> dict | None:
                 'sha256': loaded.sha256 if hasattr(loaded, 'sha256') else None,
                 'binprovider': loaded.binprovider.name if loaded.binprovider else 'env',
             }
-    except ImportError:
-        pass
     except Exception:
         pass
 
-    # Fallback to shutil.which
-    abspath = shutil.which('papers-dl') or os.environ.get('PAPERSDL_BINARY', '')
-    if abspath and Path(abspath).is_file():
-        return {
-            'name': 'papers-dl',
-            'abspath': abspath,
-            'version': get_binary_version(abspath),
-            'sha256': get_binary_hash(abspath),
-            'binprovider': 'env',
-        }
-
     return None
 
 

archivebox/plugins/readability/on_Crawl__00_validate_readability.py

@@ -6,13 +6,8 @@ Runs at crawl start to verify readability-extractor is available.
 Outputs JSONL for InstalledBinary and Machine config updates.
 """
 
-import os
 import sys
 import json
-import shutil
-import hashlib
-import subprocess
-from pathlib import Path
 
 
 def find_readability() -> dict | None:
@@ -30,22 +25,9 @@ def find_readability() -> dict | None:
                 'sha256': loaded.sha256 if hasattr(loaded, 'sha256') else None,
                 'binprovider': loaded.binprovider.name if loaded.binprovider else 'env',
             }
-    except ImportError:
-        pass
     except Exception:
         pass
 
-    # Fallback to shutil.which
-    abspath = shutil.which('readability-extractor') or os.environ.get('READABILITY_BINARY', '')
-    if abspath and Path(abspath).is_file():
-        return {
-            'name': 'readability-extractor',
-            'abspath': abspath,
-            'version': None,
-            'sha256': None,
-            'binprovider': 'env',
-        }
-
     return None
 
 

archivebox/plugins/search_backend_ripgrep/on_Crawl__00_validate_ripgrep.py

@@ -9,67 +9,25 @@ Outputs JSONL for InstalledBinary and Machine config updates.
 import os
 import sys
 import json
-import shutil
-import hashlib
-import subprocess
-from pathlib import Path
-
-
-def get_binary_version(abspath: str) -> str | None:
-    """Get version string from ripgrep binary."""
-    try:
-        result = subprocess.run(
-            [abspath, '--version'],
-            capture_output=True,
-            text=True,
-            timeout=5,
-        )
-        if result.returncode == 0 and result.stdout:
-            # ripgrep version string: "ripgrep 14.1.0"
-            first_line = result.stdout.strip().split('\n')[0]
-            parts = first_line.split()
-            for i, part in enumerate(parts):
-                if part.lower() == 'ripgrep' and i + 1 < len(parts):
-                    return parts[i + 1]
-            # Try to find version number pattern
-            for part in parts:
-                if part[0].isdigit() and '.' in part:
-                    return part
-            return first_line[:32]
-    except Exception:
-        pass
-    return None
-
-
-def get_binary_hash(abspath: str) -> str | None:
-    """Get SHA256 hash of binary."""
-    try:
-        with open(abspath, 'rb') as f:
-            return hashlib.sha256(f.read()).hexdigest()
-    except Exception:
-        return None
 
 
 def find_ripgrep() -> dict | None:
-    """Find ripgrep binary using shutil.which or env var."""
-    # Check env var first - if it's an absolute path and exists, use it
-    ripgrep_env = os.environ.get('RIPGREP_BINARY', '')
-    if ripgrep_env and '/' in ripgrep_env and Path(ripgrep_env).is_file():
-        abspath = ripgrep_env
-    else:
-        # Otherwise try shutil.which with the env var as the binary name
-        abspath = shutil.which(ripgrep_env) if ripgrep_env else None
-        if not abspath:
-            abspath = shutil.which('rg')
+    """Find ripgrep binary."""
+    try:
+        from abx_pkg import Binary, AptProvider, BrewProvider, EnvProvider
 
-    if abspath and Path(abspath).is_file():
-        return {
-            'name': 'rg',
-            'abspath': abspath,
-            'version': get_binary_version(abspath),
-            'sha256': get_binary_hash(abspath),
-            'binprovider': 'env',
-        }
+        binary = Binary(name='rg', binproviders=[AptProvider(), BrewProvider(), EnvProvider()])
+        loaded = binary.load()
+        if loaded and loaded.abspath:
+            return {
+                'name': 'rg',
+                'abspath': str(loaded.abspath),
+                'version': str(loaded.version) if loaded.version else None,
+                'sha256': loaded.sha256 if hasattr(loaded, 'sha256') else None,
+                'binprovider': loaded.binprovider.name if loaded.binprovider else 'env',
+            }
+    except Exception:
+        pass
 
     return None
 

archivebox/plugins/singlefile/on_Crawl__00_validate_singlefile.py

@@ -6,82 +6,27 @@ Runs at crawl start to verify single-file (npm package) is available.
 Outputs JSONL for InstalledBinary and Machine config updates.
 """
 
-import os
 import sys
 import json
-import shutil
-import hashlib
-import subprocess
-from pathlib import Path
-
-
-def get_binary_version(abspath: str) -> str | None:
-    """Get version string from single-file binary."""
-    try:
-        result = subprocess.run(
-            [abspath, '--version'],
-            capture_output=True,
-            text=True,
-            timeout=10,
-        )
-        if result.returncode == 0 and result.stdout:
-            return result.stdout.strip().split('\n')[0][:32]
-    except Exception:
-        pass
-    return None
-
-
-def get_binary_hash(abspath: str) -> str | None:
-    """Get SHA256 hash of binary."""
-    try:
-        # For scripts, hash the script content
-        with open(abspath, 'rb') as f:
-            return hashlib.sha256(f.read()).hexdigest()
-    except Exception:
-        return None
 
 
 def find_singlefile() -> dict | None:
     """Find single-file binary."""
-    # Check env var first
-    env_path = os.environ.get('SINGLEFILE_BINARY', '')
-    if env_path and Path(env_path).is_file():
-        return {
-            'name': 'single-file',
-            'abspath': env_path,
-            'version': get_binary_version(env_path),
-            'sha256': get_binary_hash(env_path),
-            'binprovider': 'env',
-        }
+    try:
+        from abx_pkg import Binary, NpmProvider, EnvProvider
 
-    # Try shutil.which
-    for name in ['single-file', 'singlefile']:
-        abspath = shutil.which(name)
-        if abspath:
+        binary = Binary(name='single-file', binproviders=[NpmProvider(), EnvProvider()])
+        loaded = binary.load()
+        if loaded and loaded.abspath:
             return {
                 'name': 'single-file',
-                'abspath': abspath,
-                'version': get_binary_version(abspath),
-                'sha256': get_binary_hash(abspath),
-                'binprovider': 'npm',
-            }
-
-    # Check common npm paths
-    npm_paths = [
-        Path.home() / '.npm-global/bin/single-file',
-        Path.home() / 'node_modules/.bin/single-file',
-        Path('/usr/local/bin/single-file'),
-        Path('/usr/local/lib/node_modules/.bin/single-file'),
-    ]
-    for path in npm_paths:
-        if path.is_file():
-            return {
-                'name': 'single-file',
-                'abspath': str(path),
-                'version': get_binary_version(str(path)),
-                'sha256': get_binary_hash(str(path)),
-                'binprovider': 'npm',
+                'abspath': str(loaded.abspath),
+                'version': str(loaded.version) if loaded.version else None,
+                'sha256': loaded.sha256 if hasattr(loaded, 'sha256') else None,
+                'binprovider': loaded.binprovider.name if loaded.binprovider else 'env',
             }
+    except Exception:
+        pass
 
     return None
 

archivebox/plugins/wget/on_Crawl__00_validate_wget.py

@@ -6,58 +6,16 @@ Runs at crawl start to verify wget is available.
 Outputs JSONL for InstalledBinary and Machine config updates.
 """
 
-import os
 import sys
 import json
-import shutil
-import hashlib
-import subprocess
-from pathlib import Path
-
-
-def get_binary_version(abspath: str) -> str | None:
-    """Get version string from binary."""
-    try:
-        result = subprocess.run(
-            [abspath, '--version'],
-            capture_output=True,
-            text=True,
-            timeout=5,
-        )
-        if result.returncode == 0 and result.stdout:
-            # wget version string: "GNU Wget 1.24.5 built on ..."
-            first_line = result.stdout.strip().split('\n')[0]
-            # Extract version number
-            parts = first_line.split()
-            for i, part in enumerate(parts):
-                if part.lower() == 'wget' and i + 1 < len(parts):
-                    return parts[i + 1]
-            return first_line[:32]
-    except Exception:
-        pass
-    return None
-
-
-def get_binary_hash(abspath: str) -> str | None:
-    """Get SHA256 hash of binary."""
-    try:
-        with open(abspath, 'rb') as f:
-            return hashlib.sha256(f.read()).hexdigest()
-    except Exception:
-        return None
 
 
 def find_wget() -> dict | None:
-    """Find wget binary using abx-pkg or fallback to shutil.which."""
-    # Try abx-pkg first
+    """Find wget binary using abx-pkg."""
     try:
         from abx_pkg import Binary, EnvProvider
 
-        class WgetBinary(Binary):
-            name: str = 'wget'
-            binproviders_supported = [EnvProvider()]
-
-        binary = WgetBinary()
+        binary = Binary(name='wget', binproviders=[EnvProvider()])
         loaded = binary.load()
         if loaded and loaded.abspath:
             return {
@@ -67,22 +25,9 @@ def find_wget() -> dict | None:
                 'sha256': loaded.sha256 if hasattr(loaded, 'sha256') else None,
                 'binprovider': loaded.binprovider.name if loaded.binprovider else 'env',
             }
-    except ImportError:
-        pass
     except Exception:
         pass
 
-    # Fallback to shutil.which
-    abspath = shutil.which('wget') or os.environ.get('WGET_BINARY', '')
-    if abspath and Path(abspath).is_file():
-        return {
-            'name': 'wget',
-            'abspath': abspath,
-            'version': get_binary_version(abspath),
-            'sha256': get_binary_hash(abspath),
-            'binprovider': 'env',
-        }
-
     return None