remove huey

2026-01-03 01:15:57 +10:00 · 2025-12-24 23:40:18 -08:00
parent 6c769d831c
commit d95f0dc186
105 changed files with 3635 additions and 1402 deletions
--- a/archivebox/plugins/mercury/on_Crawl__00_install_mercury.py
+++ b/archivebox/plugins/mercury/on_Crawl__00_install_mercury.py
@@ -1,68 +0,0 @@
-#!/usr/bin/env python3
-"""
-Install mercury-parser if not already available.
-
-Runs at crawl start to ensure mercury-parser is installed.
-Outputs JSONL for InstalledBinary.
-"""
-
-import json
-import sys
-from pathlib import Path
-
-
-def main():
-    try:
-        from abx_pkg import Binary, NpmProvider, EnvProvider, BinProviderOverrides
-
-        NpmProvider.model_rebuild()
-        EnvProvider.model_rebuild()
-
-        # Note: npm package is @postlight/mercury-parser, binary is mercury-parser
-        mercury_binary = Binary(
-            name='mercury-parser',
-            binproviders=[NpmProvider(), EnvProvider()],
-            overrides={'npm': {'packages': ['@postlight/mercury-parser']}}
-        )
-
-        # Try to load, install if not found
-        try:
-            loaded = mercury_binary.load()
-            if not loaded or not loaded.abspath:
-                raise Exception("Not loaded")
-        except Exception:
-            # Install via npm
-            loaded = mercury_binary.install()
-
-        if loaded and loaded.abspath:
-            # Output InstalledBinary JSONL
-            print(json.dumps({
-                'type': 'InstalledBinary',
-                'name': 'mercury-parser',
-                'abspath': str(loaded.abspath),
-                'version': str(loaded.version) if loaded.version else None,
-                'sha256': loaded.sha256,
-                'binprovider': loaded.loaded_binprovider.name if loaded.loaded_binprovider else 'unknown',
-            }))
-            sys.exit(0)
-        else:
-            print(json.dumps({
-                'type': 'Dependency',
-                'bin_name': 'mercury-parser',
-                'bin_providers': 'npm,env',
-            }))
-            print("Failed to install mercury-parser", file=sys.stderr)
-            sys.exit(1)
-
-    except Exception as e:
-        print(json.dumps({
-            'type': 'Dependency',
-            'bin_name': 'mercury-parser',
-            'bin_providers': 'npm,env',
-        }))
-        print(f"Error installing mercury-parser: {e}", file=sys.stderr)
-        sys.exit(1)
-
-
-if __name__ == '__main__':
-    main()
--- a/archivebox/plugins/mercury/on_Crawl__00_validate_mercury.py
+++ b/archivebox/plugins/mercury/on_Crawl__00_validate_mercury.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+"""
+Validation hook for postlight-parser binary.
+
+Runs at crawl start to verify postlight-parser is available.
+Outputs JSONL for InstalledBinary and Machine config updates.
+"""
+
+import os
+import sys
+import json
+import shutil
+import hashlib
+import subprocess
+from pathlib import Path
+
+
+def get_binary_version(abspath: str) -> str | None:
+    """Get version string from binary."""
+    try:
+        result = subprocess.run(
+            [abspath, '--version'],
+            capture_output=True,
+            text=True,
+            timeout=5,
+        )
+        if result.returncode == 0 and result.stdout:
+            first_line = result.stdout.strip().split('\n')[0]
+            return first_line[:64]
+    except Exception:
+        pass
+    return None
+
+
+def get_binary_hash(abspath: str) -> str | None:
+    """Get SHA256 hash of binary."""
+    try:
+        with open(abspath, 'rb') as f:
+            return hashlib.sha256(f.read()).hexdigest()
+    except Exception:
+        return None
+
+
+def find_mercury() -> dict | None:
+    """Find postlight-parser binary."""
+    try:
+        from abx_pkg import Binary, NpmProvider, EnvProvider
+
+        class MercuryBinary(Binary):
+            name: str = 'postlight-parser'
+            binproviders_supported = [NpmProvider(), EnvProvider()]
+            overrides: dict = {'npm': {'packages': ['@postlight/parser']}}
+
+        binary = MercuryBinary()
+        loaded = binary.load()
+        if loaded and loaded.abspath:
+            return {
+                'name': 'postlight-parser',
+                'abspath': str(loaded.abspath),
+                'version': str(loaded.version) if loaded.version else None,
+                'sha256': loaded.sha256 if hasattr(loaded, 'sha256') else None,
+                'binprovider': loaded.binprovider.name if loaded.binprovider else 'env',
+            }
+    except ImportError:
+        pass
+    except Exception:
+        pass
+
+    # Fallback to shutil.which
+    abspath = shutil.which('postlight-parser') or os.environ.get('MERCURY_BINARY', '')
+    if abspath and Path(abspath).is_file():
+        return {
+            'name': 'postlight-parser',
+            'abspath': abspath,
+            'version': get_binary_version(abspath),
+            'sha256': get_binary_hash(abspath),
+            'binprovider': 'env',
+        }
+
+    return None
+
+
+def main():
+    result = find_mercury()
+
+    if result and result.get('abspath'):
+        print(json.dumps({
+            'type': 'InstalledBinary',
+            'name': result['name'],
+            'abspath': result['abspath'],
+            'version': result['version'],
+            'sha256': result['sha256'],
+            'binprovider': result['binprovider'],
+        }))
+
+        print(json.dumps({
+            'type': 'Machine',
+            '_method': 'update',
+            'key': 'config/MERCURY_BINARY',
+            'value': result['abspath'],
+        }))
+
+        if result['version']:
+            print(json.dumps({
+                'type': 'Machine',
+                '_method': 'update',
+                'key': 'config/MERCURY_VERSION',
+                'value': result['version'],
+            }))
+
+        sys.exit(0)
+    else:
+        print(json.dumps({
+            'type': 'Dependency',
+            'bin_name': 'postlight-parser',
+            'bin_providers': 'npm,env',
+        }))
+        print(f"postlight-parser binary not found", file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()
--- a/archivebox/plugins/mercury/on_Snapshot__53_mercury.py
+++ b/archivebox/plugins/mercury/on_Snapshot__53_mercury.py
@@ -6,10 +6,10 @@ Usage: on_Snapshot__mercury.py --url=<url> --snapshot-id=<uuid>
 Output: Creates mercury/ directory with content.html, content.txt, article.json

 Environment variables:
-    MERCURY_BINARY: Path to mercury-parser binary
+    MERCURY_BINARY: Path to postlight-parser binary
    TIMEOUT: Timeout in seconds (default: 60)

-Note: Requires mercury-parser: npm install -g @postlight/mercury-parser
+Note: Requires postlight-parser: npm install -g @postlight/parser
 """

 import json
@@ -25,7 +25,7 @@ import rich_click as click

 # Extractor metadata
 EXTRACTOR_NAME = 'mercury'
-BIN_NAME = 'mercury-parser'
+BIN_NAME = 'postlight-parser'
 BIN_PROVIDERS = 'npm,env'
 OUTPUT_DIR = 'mercury'

@@ -42,12 +42,12 @@ def get_env_int(name: str, default: int = 0) -> int:


 def find_mercury() -> str | None:
-    """Find mercury-parser binary."""
+    """Find postlight-parser binary."""
    mercury = get_env('MERCURY_BINARY')
    if mercury and os.path.isfile(mercury):
        return mercury

-    for name in ['mercury-parser', 'mercury']:
+    for name in ['postlight-parser']:
        binary = shutil.which(name)
        if binary:
            return binary
@@ -56,7 +56,7 @@ def find_mercury() -> str | None:


 def get_version(binary: str) -> str:
-    """Get mercury-parser version."""
+    """Get postlight-parser version."""
    try:
        result = subprocess.run([binary, '--version'], capture_output=True, text=True, timeout=10)
        return result.stdout.strip()[:64]
@@ -83,12 +83,12 @@ def extract_mercury(url: str, binary: str) -> tuple[bool, str | None, str]:

        if result_text.returncode != 0:
            stderr = result_text.stderr.decode('utf-8', errors='replace')
-            return False, None, f'mercury-parser failed: {stderr[:200]}'
+            return False, None, f'postlight-parser failed: {stderr[:200]}'

        try:
            text_json = json.loads(result_text.stdout)
        except json.JSONDecodeError:
-            return False, None, 'mercury-parser returned invalid JSON'
+            return False, None, 'postlight-parser returned invalid JSON'

        if text_json.get('failed'):
            return False, None, 'Mercury was not able to extract article'
@@ -139,7 +139,7 @@ def main(url: str, snapshot_id: str):
        # Find binary
        binary = find_mercury()
        if not binary:
-            print(f'ERROR: mercury-parser binary not found', file=sys.stderr)
+            print(f'ERROR: postlight-parser binary not found', file=sys.stderr)
            print(f'DEPENDENCY_NEEDED={BIN_NAME}', file=sys.stderr)
            print(f'BIN_PROVIDERS={BIN_PROVIDERS}', file=sys.stderr)
            sys.exit(1)
--- a/archivebox/plugins/mercury/templates/embed.html
+++ b/archivebox/plugins/mercury/templates/embed.html
@@ -0,0 +1,6 @@
+<!-- Mercury embed - Mercury parser article view -->
+<iframe src="{{ output_path }}"
+        class="extractor-embed mercury-embed"
+        style="width: 100%; height: 100%; min-height: 500px; border: none; background: #fefefe;"
+        sandbox="allow-same-origin">
+</iframe>
--- a/archivebox/plugins/mercury/templates/fullscreen.html
+++ b/archivebox/plugins/mercury/templates/fullscreen.html
@@ -0,0 +1,6 @@
+<!-- Mercury fullscreen - full Mercury parser article -->
+<iframe src="{{ output_path }}"
+        class="extractor-fullscreen mercury-fullscreen"
+        style="width: 100%; height: 100vh; border: none; background: #fefefe;"
+        sandbox="allow-same-origin">
+</iframe>
--- a/archivebox/plugins/mercury/templates/icon.html
+++ b/archivebox/plugins/mercury/templates/icon.html
@@ -0,0 +1 @@
+☿️
--- a/archivebox/plugins/mercury/templates/thumbnail.html
+++ b/archivebox/plugins/mercury/templates/thumbnail.html
@@ -0,0 +1,8 @@
+<!-- Mercury thumbnail - shows Mercury parser extracted article content -->
+<div class="extractor-thumbnail mercury-thumbnail" style="width: 100%; height: 100px; overflow: hidden; background: #fefefe; padding: 8px; font-family: Georgia, serif; font-size: 11px; line-height: 1.4; color: #333;">
+    <iframe src="{{ output_path }}"
+            style="width: 100%; height: 300px; border: none; pointer-events: none;"
+            loading="lazy"
+            sandbox="allow-same-origin">
+    </iframe>
+</div>
--- a/archivebox/plugins/mercury/tests/test_mercury.py
+++ b/archivebox/plugins/mercury/tests/test_mercury.py
@@ -21,7 +21,7 @@ import pytest
 PLUGIN_DIR = Path(__file__).parent.parent
 PLUGINS_ROOT = PLUGIN_DIR.parent
 MERCURY_HOOK = PLUGIN_DIR / 'on_Snapshot__53_mercury.py'
-MERCURY_INSTALL_HOOK = PLUGIN_DIR / 'on_Crawl__00_install_mercury.py'
+MERCURY_VALIDATE_HOOK = PLUGIN_DIR / 'on_Crawl__00_validate_mercury.py'
 TEST_URL = 'https://example.com'

 def test_hook_script_exists():
@@ -29,53 +29,70 @@ def test_hook_script_exists():
    assert MERCURY_HOOK.exists(), f"Hook not found: {MERCURY_HOOK}"


-def test_mercury_install_hook():
-    """Test mercury install hook to install mercury-parser if needed."""
-    # Run mercury install hook
+def test_mercury_validate_hook():
+    """Test mercury validate hook checks for postlight-parser."""
+    # Run mercury validate hook
    result = subprocess.run(
-        [sys.executable, str(MERCURY_INSTALL_HOOK)],
+        [sys.executable, str(MERCURY_VALIDATE_HOOK)],
        capture_output=True,
        text=True,
-        timeout=600
+        timeout=30
    )

-    assert result.returncode == 0, f"Install hook failed: {result.stderr}"
-
-    # Verify InstalledBinary JSONL output
-    found_binary = False
-    for line in result.stdout.strip().split('\n'):
-        if line.strip():
-            try:
-                record = json.loads(line)
-                if record.get('type') == 'InstalledBinary':
-                    assert record['name'] == 'mercury-parser'
-                    assert record['abspath']
-                    found_binary = True
-                    break
-            except json.JSONDecodeError:
-                pass
-
-    assert found_binary, "Should output InstalledBinary record"
+    # Hook exits 0 if binary found, 1 if not found (with Dependency record)
+    if result.returncode == 0:
+        # Binary found - verify InstalledBinary JSONL output
+        found_binary = False
+        for line in result.stdout.strip().split('\n'):
+            if line.strip():
+                try:
+                    record = json.loads(line)
+                    if record.get('type') == 'InstalledBinary':
+                        assert record['name'] == 'postlight-parser'
+                        assert record['abspath']
+                        found_binary = True
+                        break
+                except json.JSONDecodeError:
+                    pass
+        assert found_binary, "Should output InstalledBinary record when binary found"
+    else:
+        # Binary not found - verify Dependency JSONL output
+        found_dependency = False
+        for line in result.stdout.strip().split('\n'):
+            if line.strip():
+                try:
+                    record = json.loads(line)
+                    if record.get('type') == 'Dependency':
+                        assert record['bin_name'] == 'postlight-parser'
+                        assert 'npm' in record['bin_providers']
+                        found_dependency = True
+                        break
+                except json.JSONDecodeError:
+                    pass
+        assert found_dependency, "Should output Dependency record when binary not found"


 def test_verify_deps_with_abx_pkg():
-    """Verify mercury-parser is available via abx-pkg after hook installation."""
+    """Verify postlight-parser is available via abx-pkg."""
    from abx_pkg import Binary, NpmProvider, EnvProvider, BinProviderOverrides

-    NpmProvider.model_rebuild()
-    EnvProvider.model_rebuild()
-
-    # Verify mercury-parser is available
+    # Verify postlight-parser is available
    mercury_binary = Binary(
-        name='mercury-parser',
+        name='postlight-parser',
        binproviders=[NpmProvider(), EnvProvider()],
-        overrides={'npm': {'packages': ['@postlight/mercury-parser']}}
+        overrides={'npm': {'packages': ['@postlight/parser']}}
    )
    mercury_loaded = mercury_binary.load()
-    assert mercury_loaded and mercury_loaded.abspath, "mercury-parser should be available after install hook"
+
+    # If validate hook found it (exit 0), this should succeed
+    # If validate hook didn't find it (exit 1), this may fail unless binprovider installed it
+    if mercury_loaded and mercury_loaded.abspath:
+        assert True, "postlight-parser is available"
+    else:
+        pytest.skip("postlight-parser not available - Dependency record should have been emitted")

 def test_extracts_with_mercury_parser():
-    """Test full workflow: extract with mercury-parser from real HTML via hook."""
+    """Test full workflow: extract with postlight-parser from real HTML via hook."""
    # Prerequisites checked by earlier test

    with tempfile.TemporaryDirectory() as tmpdir: