wip 2

archivebox/plugins/readability/on_Crawl__00_install_readability.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+"""
+Install readability-extractor if not already available.
+
+Runs at crawl start to ensure readability-extractor is installed.
+Outputs JSONL for InstalledBinary.
+"""
+
+import json
+import sys
+from pathlib import Path
+
+
+def main():
+    try:
+        from abx_pkg import Binary, NpmProvider, EnvProvider, BinProviderOverrides
+
+        NpmProvider.model_rebuild()
+        EnvProvider.model_rebuild()
+
+        # Note: npm package is from github:ArchiveBox/readability-extractor
+        readability_binary = Binary(
+            name='readability-extractor',
+            binproviders=[NpmProvider(), EnvProvider()],
+            overrides={'npm': {'packages': ['github:ArchiveBox/readability-extractor']}}
+        )
+
+        # Try to load, install if not found
+        try:
+            loaded = readability_binary.load()
+            if not loaded or not loaded.abspath:
+                raise Exception("Not loaded")
+        except Exception:
+            # Install via npm from GitHub repo
+            loaded = readability_binary.install()
+
+        if loaded and loaded.abspath:
+            # Output InstalledBinary JSONL
+            print(json.dumps({
+                'type': 'InstalledBinary',
+                'name': 'readability-extractor',
+                'abspath': str(loaded.abspath),
+                'version': str(loaded.version) if loaded.version else None,
+                'sha256': loaded.sha256,
+                'binprovider': loaded.loaded_binprovider.name if loaded.loaded_binprovider else 'unknown',
+            }))
+            sys.exit(0)
+        else:
+            print(json.dumps({
+                'type': 'Dependency',
+                'bin_name': 'readability-extractor',
+                'bin_providers': 'npm,env',
+            }))
+            print("Failed to install readability-extractor", file=sys.stderr)
+            sys.exit(1)
+
+    except Exception as e:
+        print(json.dumps({
+            'type': 'Dependency',
+            'bin_name': 'readability-extractor',
+            'bin_providers': 'npm,env',
+        }))
+        print(f"Error installing readability-extractor: {e}", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

archivebox/plugins/readability/on_Snapshot__52_readability.py

@@ -6,10 +6,10 @@ Usage: on_Snapshot__readability.py --url=<url> --snapshot-id=<uuid>
 Output: Creates readability/ directory with content.html, content.txt, article.json
 
 Environment variables:
-    READABILITY_BINARY: Path to readability-cli binary
+    READABILITY_BINARY: Path to readability-extractor binary
     TIMEOUT: Timeout in seconds (default: 60)
 
-Note: Requires readability-cli: npm install -g readability-cli
+Note: Requires readability-extractor from https://github.com/ArchiveBox/readability-extractor
       This extractor looks for HTML source from other extractors (wget, singlefile, dom)
 """
 
@@ -27,7 +27,7 @@ import rich_click as click
 
 # Extractor metadata
 EXTRACTOR_NAME = 'readability'
-BIN_NAME = 'readability-cli'
+BIN_NAME = 'readability-extractor'
 BIN_PROVIDERS = 'npm,env'
 OUTPUT_DIR = 'readability'
 
@@ -44,12 +44,12 @@ def get_env_int(name: str, default: int = 0) -> int:
 
 
 def find_readability() -> str | None:
-    """Find readability-cli binary."""
+    """Find readability-extractor binary."""
     readability = get_env('READABILITY_BINARY')
     if readability and os.path.isfile(readability):
         return readability
 
-    for name in ['readability-cli', 'readable']:
+    for name in ['readability-extractor']:
         binary = shutil.which(name)
         if binary:
             return binary
@@ -58,7 +58,7 @@ def find_readability() -> str | None:
 
 
 def get_version(binary: str) -> str:
-    """Get readability-cli version."""
+    """Get readability-extractor version."""
     try:
         result = subprocess.run([binary, '--version'], capture_output=True, text=True, timeout=10)
         return result.stdout.strip()[:64]
@@ -106,24 +106,24 @@ def extract_readability(url: str, binary: str) -> tuple[bool, str | None, str]:
     output_dir.mkdir(exist_ok=True)
 
     try:
-        # Run readability-cli
-        cmd = [binary, '--json', html_source]
+        # Run readability-extractor (outputs JSON by default)
+        cmd = [binary, html_source]
         result = subprocess.run(cmd, capture_output=True, timeout=timeout)
 
         if result.returncode != 0:
             stderr = result.stderr.decode('utf-8', errors='replace')
-            return False, None, f'readability-cli failed: {stderr[:200]}'
+            return False, None, f'readability-extractor failed: {stderr[:200]}'
 
         # Parse JSON output
         try:
             result_json = json.loads(result.stdout)
         except json.JSONDecodeError:
-            return False, None, 'readability-cli returned invalid JSON'
+            return False, None, 'readability-extractor returned invalid JSON'
 
         # Extract and save content
-        # readability-cli v2.x uses hyphenated field names
-        text_content = result_json.pop('text-content', result_json.pop('textContent', ''))
-        html_content = result_json.pop('html-content', result_json.pop('content', ''))
+        # readability-extractor uses camelCase field names (textContent, content)
+        text_content = result_json.pop('textContent', result_json.pop('text-content', ''))
+        html_content = result_json.pop('content', result_json.pop('html-content', ''))
 
         if not text_content and not html_content:
             return False, None, 'No content extracted'
@@ -157,7 +157,7 @@ def main(url: str, snapshot_id: str):
         # Find binary
         binary = find_readability()
         if not binary:
-            print(f'ERROR: readability-cli binary not found', file=sys.stderr)
+            print(f'ERROR: readability-extractor binary not found', file=sys.stderr)
             print(f'DEPENDENCY_NEEDED={BIN_NAME}', file=sys.stderr)
             print(f'BIN_PROVIDERS={BIN_PROVIDERS}', file=sys.stderr)
             sys.exit(1)
@@ -187,7 +187,7 @@ def main(url: str, snapshot_id: str):
     print(f'END_TS={end_ts.isoformat()}')
     print(f'DURATION={duration:.2f}')
     if binary:
-        print(f'CMD={binary} --json <html>')
+        print(f'CMD={binary} <html>')
     if version:
         print(f'VERSION={version}')
     if output:

archivebox/plugins/readability/tests/test_readability.py

@@ -2,9 +2,10 @@
 Integration tests for readability plugin
 
 Tests verify:
-1. Plugin reports missing dependency correctly
-2. readability-cli can be installed via npm (note: package name != binary name)
-3. Extraction works against real example.com content
+1. Install hook installs readability-extractor via abx-pkg
+2. Verify deps with abx-pkg
+3. Plugin reports missing dependency correctly
+4. Extraction works against real example.com content
 """
 
 import json
@@ -20,6 +21,7 @@ import pytest
 PLUGIN_DIR = Path(__file__).parent.parent
 PLUGINS_ROOT = PLUGIN_DIR.parent
 READABILITY_HOOK = next(PLUGIN_DIR.glob('on_Snapshot__*_readability.py'))
+READABILITY_INSTALL_HOOK = PLUGIN_DIR / 'on_Crawl__00_install_readability.py'
 TEST_URL = 'https://example.com'
 
 
@@ -74,7 +76,7 @@ def test_hook_script_exists():
 
 
 def test_reports_missing_dependency_when_not_installed():
-    """Test that script reports DEPENDENCY_NEEDED when readability-cli is not found."""
+    """Test that script reports DEPENDENCY_NEEDED when readability-extractor is not found."""
     with tempfile.TemporaryDirectory() as tmpdir:
         tmpdir = Path(tmpdir)
 
@@ -96,68 +98,57 @@ def test_reports_missing_dependency_when_not_installed():
         assert result.returncode != 0, "Should exit non-zero when dependency missing"
         combined = result.stdout + result.stderr
         assert 'DEPENDENCY_NEEDED' in combined, "Should output DEPENDENCY_NEEDED"
-        assert 'readability-cli' in combined or 'BIN_NAME' in combined, "Should mention readability-cli"
+        assert 'readability-extractor' in combined or 'BIN_NAME' in combined, "Should mention readability-extractor"
 
 
-def test_can_install_readability_via_npm():
-    """Test that readability-cli can be installed via npm and binary becomes available.
-
-    Note: The npm package 'readability-cli' installs a binary named 'readable',
-    so we test the full installation flow using npm install directly.
-    """
-
-    # Check npm is available
-    if not shutil.which('npm'):
-        pytest.skip("npm not available on this system")
-
-    # Install readability-cli package via npm
-    # The orchestrator/dependency hooks would call this via npm provider
+def test_readability_install_hook():
+    """Test readability install hook to install readability-extractor if needed."""
     result = subprocess.run(
-        ['npm', 'install', '-g', 'readability-cli'],
+        [sys.executable, str(READABILITY_INSTALL_HOOK)],
         capture_output=True,
         text=True,
-        timeout=300
+        timeout=600
     )
 
-    assert result.returncode == 0, f"npm install failed: {result.stderr}"
+    assert result.returncode == 0, f"Install hook failed: {result.stderr}"
 
-    # Verify the 'readable' binary is now available
-    # (readability-cli package installs as 'readable' not 'readability-cli')
-    result = subprocess.run(['which', 'readable'], capture_output=True, text=True)
-    assert result.returncode == 0, "readable binary not found after npm install"
+    # Verify InstalledBinary JSONL output
+    found_binary = False
+    for line in result.stdout.strip().split('\n'):
+        if line.strip():
+            try:
+                record = json.loads(line)
+                if record.get('type') == 'InstalledBinary':
+                    assert record['name'] == 'readability-extractor'
+                    assert record['abspath']
+                    found_binary = True
+                    break
+            except json.JSONDecodeError:
+                pass
 
-    binary_path = result.stdout.strip()
-    assert Path(binary_path).exists(), f"Binary should exist at {binary_path}"
+    assert found_binary, "Should output InstalledBinary record"
 
-    # Test that it's executable and responds to --version
-    result = subprocess.run(
-        [binary_path, '--version'],
-        capture_output=True,
-        text=True,
-        timeout=10
+
+def test_verify_deps_with_abx_pkg():
+    """Verify readability-extractor is available via abx-pkg after hook installation."""
+    from abx_pkg import Binary, NpmProvider, EnvProvider, BinProviderOverrides
+
+    NpmProvider.model_rebuild()
+    EnvProvider.model_rebuild()
+
+    readability_binary = Binary(
+        name='readability-extractor',
+        binproviders=[NpmProvider(), EnvProvider()],
+        overrides={'npm': {'packages': ['github:ArchiveBox/readability-extractor']}}
     )
-    assert result.returncode == 0, f"Binary not executable: {result.stderr}"
+    readability_loaded = readability_binary.load()
+    assert readability_loaded and readability_loaded.abspath, "readability-extractor should be available after install hook"
 
 
 def test_extracts_article_after_installation():
-    """Test full workflow: ensure readability-cli installed then extract from example.com HTML."""
+    """Test full workflow: extract article using readability-extractor from real HTML."""
+    # Prerequisites checked by earlier test (install hook should have run)
 
-    # Check npm is available
-    if not shutil.which('npm'):
-        pytest.skip("npm not available on this system")
-
-    # Ensure readability-cli is installed (orchestrator would handle this)
-    install_result = subprocess.run(
-        ['npm', 'install', '-g', 'readability-cli'],
-        capture_output=True,
-        text=True,
-        timeout=300
-    )
-
-    if install_result.returncode != 0:
-        pytest.skip(f"Could not install readability-cli: {install_result.stderr}")
-
-    # Now test extraction
     with tempfile.TemporaryDirectory() as tmpdir:
         tmpdir = Path(tmpdir)
 
@@ -213,21 +204,7 @@ def test_extracts_article_after_installation():
 
 def test_fails_gracefully_without_html_source():
     """Test that extraction fails gracefully when no HTML source is available."""
-
-    # Check npm is available
-    if not shutil.which('npm'):
-        pytest.skip("npm not available on this system")
-
-    # Ensure readability-cli is installed
-    install_result = subprocess.run(
-        ['npm', 'install', '-g', 'readability-cli'],
-        capture_output=True,
-        text=True,
-        timeout=300
-    )
-
-    if install_result.returncode != 0:
-        pytest.skip("Could not install readability-cli")
+    # Prerequisites checked by earlier test (install hook should have run)
 
     with tempfile.TemporaryDirectory() as tmpdir:
         tmpdir = Path(tmpdir)