Clean up on_Crawl hooks: remove duplicates and standardize naming

Deleted dead/duplicate hooks:
- wget/on_Crawl__10_install_wget.py (duplicate of __10_wget_validate_config.py)
- chrome/on_Crawl__00_chrome_install.py (simpler version, kept full one)
- chrome/on_Crawl__20_chrome_launch.bg.js (legacy, kept __30 version)
- singlefile/on_Crawl__20_install_singlefile_extension.js (disabled/dead)
- istilldontcareaboutcookies/on_Crawl__20_install_*.js (legacy)
- ublock/on_Crawl__03_ublock.js (legacy, kept __20 version)
- Entire captcha2/ plugin (legacy version of twocaptcha/)

Renamed hooks to follow consistent pattern: on_Crawl__XX_<plugin>_<action>.<ext>
Priority bands:
  00-09: Binary/extension installation
  10-19: Config validation
  20-29: Browser launch and post-launch config

Final hooks:
  00 ripgrep_install.py, 01 chrome_install.py
  02 istilldontcareaboutcookies_install.js
  03 ublock_install.js, 04 singlefile_install.js
  05 twocaptcha_install.js
  10 chrome_validate.py, 11 wget_validate.py
  20 chrome_launch.bg.js, 25 twocaptcha_config.js

archivebox/plugins/chrome/on_Crawl__00_chrome_install.py

@@ -1,184 +0,0 @@
-#!/usr/bin/env python3
-"""
-Install hook for Chrome/Chromium and puppeteer-core.
-
-Runs at crawl start to install/find Chromium and puppeteer-core.
-Outputs JSONL for Binary and Machine config updates.
-Respects CHROME_BINARY env var for custom binary paths.
-Uses `npx @puppeteer/browsers install chromium@latest` and parses output.
-
-NOTE: We use Chromium instead of Chrome because Chrome 137+ removed support for
---load-extension and --disable-extensions-except flags, which are needed for
-loading unpacked extensions in headless mode.
-"""
-
-import os
-import sys
-import json
-import subprocess
-from pathlib import Path
-
-
-def get_chrome_version(binary_path: str) -> str | None:
-    """Get Chrome/Chromium version string."""
-    try:
-        result = subprocess.run(
-            [binary_path, '--version'],
-            capture_output=True,
-            text=True,
-            timeout=5
-        )
-        if result.returncode == 0:
-            return result.stdout.strip()
-    except Exception:
-        pass
-    return None
-
-
-def install_puppeteer_core() -> bool:
-    """Install puppeteer-core to NODE_MODULES_DIR if not present."""
-    node_modules_dir = os.environ.get('NODE_MODULES_DIR', '').strip()
-    if not node_modules_dir:
-        # No isolated node_modules, skip (will use global)
-        return True
-
-    node_modules_path = Path(node_modules_dir)
-    if (node_modules_path / 'puppeteer-core').exists():
-        return True
-
-    # Get npm prefix from NODE_MODULES_DIR (parent of node_modules)
-    npm_prefix = node_modules_path.parent
-
-    try:
-        print(f"[*] Installing puppeteer-core to {npm_prefix}...", file=sys.stderr)
-        result = subprocess.run(
-            ['npm', 'install', '--prefix', str(npm_prefix), 'puppeteer-core', '@puppeteer/browsers'],
-            capture_output=True,
-            text=True,
-            timeout=60
-        )
-        if result.returncode == 0:
-            print(f"[+] puppeteer-core installed", file=sys.stderr)
-            return True
-        else:
-            print(f"[!] Failed to install puppeteer-core: {result.stderr}", file=sys.stderr)
-            return False
-    except Exception as e:
-        print(f"[!] Failed to install puppeteer-core: {e}", file=sys.stderr)
-        return False
-
-
-def install_chromium() -> dict | None:
-    """Install Chromium using @puppeteer/browsers and parse output for binary path.
-
-    Output format: "chromium@<version> <path_to_binary>"
-    e.g.: "chromium@1563294 /Users/x/.cache/puppeteer/chromium/.../Chromium"
-
-    Note: npx is fast when chromium is already cached - it returns the path without re-downloading.
-    """
-    try:
-        print("[*] Installing Chromium via @puppeteer/browsers...", file=sys.stderr)
-
-        # Use --path to install to puppeteer's standard cache location
-        cache_path = os.path.expanduser('~/.cache/puppeteer')
-
-        result = subprocess.run(
-            ['npx', '@puppeteer/browsers', 'install', 'chromium@1563297', f'--path={cache_path}'],
-            capture_output=True,
-            text=True,
-            stdin=subprocess.DEVNULL,
-            timeout=300
-        )
-
-        if result.returncode != 0:
-            print(f"[!] Failed to install Chromium: {result.stderr}", file=sys.stderr)
-            return None
-
-        # Parse output: "chromium@1563294 /path/to/Chromium"
-        output = result.stdout.strip()
-        parts = output.split(' ', 1)
-        if len(parts) != 2:
-            print(f"[!] Failed to parse install output: {output}", file=sys.stderr)
-            return None
-
-        version_str = parts[0]  # "chromium@1563294"
-        binary_path = parts[1].strip()
-
-        if not binary_path or not os.path.exists(binary_path):
-            print(f"[!] Binary not found at: {binary_path}", file=sys.stderr)
-            return None
-
-        # Extract version number
-        version = version_str.split('@')[1] if '@' in version_str else None
-
-        print(f"[+] Chromium installed: {binary_path}", file=sys.stderr)
-
-        return {
-            'name': 'chromium',
-            'abspath': binary_path,
-            'version': version,
-            'binprovider': 'puppeteer',
-        }
-
-    except subprocess.TimeoutExpired:
-        print("[!] Chromium install timed out", file=sys.stderr)
-    except FileNotFoundError:
-        print("[!] npx not found - is Node.js installed?", file=sys.stderr)
-    except Exception as e:
-        print(f"[!] Failed to install Chromium: {e}", file=sys.stderr)
-
-    return None
-
-
-def main():
-    # Install puppeteer-core if NODE_MODULES_DIR is set
-    install_puppeteer_core()
-
-    # Check if CHROME_BINARY is already set and valid
-    configured_binary = os.environ.get('CHROME_BINARY', '').strip()
-    if configured_binary and os.path.isfile(configured_binary) and os.access(configured_binary, os.X_OK):
-        version = get_chrome_version(configured_binary)
-        print(json.dumps({
-            'type': 'Binary',
-            'name': 'chromium',
-            'abspath': configured_binary,
-            'version': version,
-            'binprovider': 'env',
-        }))
-        sys.exit(0)
-
-    # Install/find Chromium via puppeteer
-    result = install_chromium()
-
-    if result and result.get('abspath'):
-        print(json.dumps({
-            'type': 'Binary',
-            'name': result['name'],
-            'abspath': result['abspath'],
-            'version': result['version'],
-            'binprovider': result['binprovider'],
-        }))
-
-        print(json.dumps({
-            'type': 'Machine',
-            '_method': 'update',
-            'key': 'config/CHROME_BINARY',
-            'value': result['abspath'],
-        }))
-
-        if result['version']:
-            print(json.dumps({
-                'type': 'Machine',
-                '_method': 'update',
-                'key': 'config/CHROMIUM_VERSION',
-                'value': result['version'],
-            }))
-
-        sys.exit(0)
-    else:
-        print("Chromium binary not found", file=sys.stderr)
-        sys.exit(1)
-
-
-if __name__ == '__main__':
-    main()

archivebox/plugins/chrome/on_Crawl__20_chrome_launch.bg.js

@@ -8,8 +8,8 @@
  * NOTE: We use Chromium instead of Chrome because Chrome 137+ removed support for
  * --load-extension and --disable-extensions-except flags.
  *
- * Usage: on_Crawl__20_chrome_launch.bg.js --crawl-id=<uuid> --source-url=<url>
- * Output: Creates chrome/ directory under crawl output dir with:
+ * Usage: on_Crawl__30_chrome_launch.bg.js --crawl-id=<uuid> --source-url=<url>
+ * Output: Writes to current directory (executor creates chrome/ dir):
  *   - cdp_url.txt: WebSocket URL for CDP connection
  *   - chrome.pid: Chromium process ID (for cleanup)
  *   - port.txt: Debug port number
@@ -38,11 +38,12 @@ const {
     killChrome,
     getEnv,
     writePidWithMtime,
+    getExtensionsDir,
 } = require('./chrome_utils.js');
 
 // Extractor metadata
 const PLUGIN_NAME = 'chrome_launch';
-const OUTPUT_DIR = 'chrome';
+const OUTPUT_DIR = '.';
 
 // Global state for cleanup
 let chromePid = null;
@@ -115,8 +116,12 @@ async function main() {
         if (version) console.error(`[*] Version: ${version}`);
 
         // Load installed extensions
-        const extensionsDir = getEnv('CHROME_EXTENSIONS_DIR') ||
-            path.join(getEnv('DATA_DIR', '.'), 'personas', getEnv('ACTIVE_PERSONA', 'Default'), 'chrome_extensions');
+        const extensionsDir = getExtensionsDir();
+        const userDataDir = getEnv('CHROME_USER_DATA_DIR');
+
+        if (userDataDir) {
+            console.error(`[*] Using user data dir: ${userDataDir}`);
+        }
 
         const installedExtensions = [];
         const extensionPaths = [];
@@ -143,17 +148,18 @@ async function main() {
             console.error(`[+] Found ${installedExtensions.length} extension(s) to load`);
         }
 
-        // Write hook's own PID
-        const hookStartTime = Date.now() / 1000;
+        // Note: PID file is written by run_hook() with hook-specific name
+        // Snapshot.cleanup() kills all *.pid processes when done
         if (!fs.existsSync(OUTPUT_DIR)) {
             fs.mkdirSync(OUTPUT_DIR, { recursive: true });
         }
-        writePidWithMtime(path.join(OUTPUT_DIR, 'hook.pid'), process.pid, hookStartTime);
 
         // Launch Chromium using consolidated function
+        // userDataDir is derived from ACTIVE_PERSONA by get_config() if not explicitly set
         const result = await launchChromium({
             binary,
             outputDir: OUTPUT_DIR,
+            userDataDir,
             extensionPaths,
         });
 
@@ -165,14 +171,6 @@ async function main() {
         chromePid = result.pid;
         const cdpUrl = result.cdpUrl;
 
-        // Write extensions metadata
-        if (installedExtensions.length > 0) {
-            fs.writeFileSync(
-                path.join(OUTPUT_DIR, 'extensions.json'),
-                JSON.stringify(installedExtensions, null, 2)
-            );
-        }
-
         // Connect puppeteer for extension verification
         console.error(`[*] Connecting puppeteer to CDP...`);
         const browser = await puppeteer.connect({
@@ -181,30 +179,102 @@ async function main() {
         });
         browserInstance = browser;
 
-        // Verify extensions loaded
+        // Get actual extension IDs from chrome://extensions page
         if (extensionPaths.length > 0) {
-            await new Promise(r => setTimeout(r, 3000));
+            await new Promise(r => setTimeout(r, 2000));
 
-            const targets = browser.targets();
-            console.error(`[*] All browser targets (${targets.length}):`);
-            for (const t of targets) {
-                console.error(`    - ${t.type()}: ${t.url().slice(0, 80)}`);
+            try {
+                const extPage = await browser.newPage();
+                await extPage.goto('chrome://extensions', { waitUntil: 'domcontentloaded', timeout: 10000 });
+                await new Promise(r => setTimeout(r, 2000));
+
+                // Parse extension info from the page
+                const extensionsFromPage = await extPage.evaluate(() => {
+                    const extensions = [];
+                    // Extensions manager uses shadow DOM
+                    const manager = document.querySelector('extensions-manager');
+                    if (!manager || !manager.shadowRoot) return extensions;
+
+                    const itemList = manager.shadowRoot.querySelector('extensions-item-list');
+                    if (!itemList || !itemList.shadowRoot) return extensions;
+
+                    const items = itemList.shadowRoot.querySelectorAll('extensions-item');
+                    for (const item of items) {
+                        const id = item.getAttribute('id');
+                        const nameEl = item.shadowRoot?.querySelector('#name');
+                        const name = nameEl?.textContent?.trim() || '';
+                        if (id && name) {
+                            extensions.push({ id, name });
+                        }
+                    }
+                    return extensions;
+                });
+
+                console.error(`[*] Found ${extensionsFromPage.length} extension(s) on chrome://extensions`);
+                for (const e of extensionsFromPage) {
+                    console.error(`    - ${e.id}: "${e.name}"`);
+                }
+
+                // Match extensions by name (strict matching)
+                for (const ext of installedExtensions) {
+                    // Read the extension's manifest to get its display name
+                    const manifestPath = path.join(ext.unpacked_path, 'manifest.json');
+                    if (fs.existsSync(manifestPath)) {
+                        const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
+                        let manifestName = manifest.name || '';
+
+                        // Resolve message placeholder (e.g., __MSG_extName__)
+                        if (manifestName.startsWith('__MSG_') && manifestName.endsWith('__')) {
+                            const msgKey = manifestName.slice(6, -2); // Extract key from __MSG_key__
+                            const defaultLocale = manifest.default_locale || 'en';
+                            const messagesPath = path.join(ext.unpacked_path, '_locales', defaultLocale, 'messages.json');
+                            if (fs.existsSync(messagesPath)) {
+                                try {
+                                    const messages = JSON.parse(fs.readFileSync(messagesPath, 'utf-8'));
+                                    if (messages[msgKey] && messages[msgKey].message) {
+                                        manifestName = messages[msgKey].message;
+                                    }
+                                } catch (e) {
+                                    console.error(`[!] Failed to read messages.json: ${e.message}`);
+                                }
+                            }
+                        }
+
+                        console.error(`[*] Looking for match: ext.name="${ext.name}" manifest.name="${manifestName}"`);
+
+                        // Find matching extension from page by exact name match first
+                        let match = extensionsFromPage.find(e => e.name === manifestName);
+
+                        // If no exact match, try case-insensitive exact match
+                        if (!match) {
+                            match = extensionsFromPage.find(e =>
+                                e.name.toLowerCase() === manifestName.toLowerCase()
+                            );
+                        }
+
+                        if (match) {
+                            ext.id = match.id;
+                            console.error(`[+] Matched extension: ${ext.name} (${manifestName}) -> ${match.id}`);
+                        } else {
+                            console.error(`[!] No match found for: ${ext.name} (${manifestName})`);
+                        }
+                    }
+                }
+
+                await extPage.close();
+            } catch (e) {
+                console.error(`[!] Failed to get extensions from chrome://extensions: ${e.message}`);
             }
 
-            const extTargets = targets.filter(t =>
-                t.url().startsWith('chrome-extension://') ||
-                t.type() === 'service_worker' ||
-                t.type() === 'background_page'
-            );
-
-            // Filter out built-in extensions
+            // Fallback: check browser targets
+            const targets = browser.targets();
             const builtinIds = [
                 'nkeimhogjdpnpccoofpliimaahmaaome',
                 'fignfifoniblkonapihmkfakmlgkbkcf',
                 'ahfgeienlihckogmohjhadlkjgocpleb',
                 'mhjfbmdgcfjbbpaeojofohoefgiehjai',
             ];
-            const customExtTargets = extTargets.filter(t => {
+            const customExtTargets = targets.filter(t => {
                 const url = t.url();
                 if (!url.startsWith('chrome-extension://')) return false;
                 const extId = url.split('://')[1].split('/')[0];
@@ -216,7 +286,7 @@ async function main() {
             for (const target of customExtTargets) {
                 const url = target.url();
                 const extId = url.split('://')[1].split('/')[0];
-                console.error(`[+] Extension loaded: ${extId} (${target.type()})`);
+                console.error(`[+] Extension target: ${extId} (${target.type()})`);
             }
 
             if (customExtTargets.length === 0 && extensionPaths.length > 0) {
@@ -225,6 +295,14 @@ async function main() {
             }
         }
 
+        // Write extensions metadata with actual IDs
+        if (installedExtensions.length > 0) {
+            fs.writeFileSync(
+                path.join(OUTPUT_DIR, 'extensions.json'),
+                JSON.stringify(installedExtensions, null, 2)
+            );
+        }
+
         console.error(`[+] Chromium session started for crawl ${crawlId}`);
         console.error(`[+] CDP URL: ${cdpUrl}`);
         console.error(`[+] PID: ${chromePid}`);

archivebox/plugins/chrome/on_Crawl__30_chrome_launch.bg.js

@@ -1,323 +0,0 @@
-#!/usr/bin/env node
-/**
- * Launch a shared Chromium browser session for the entire crawl.
- *
- * This runs once per crawl and keeps Chromium alive for all snapshots to share.
- * Each snapshot creates its own tab via on_Snapshot__20_chrome_tab.bg.js.
- *
- * NOTE: We use Chromium instead of Chrome because Chrome 137+ removed support for
- * --load-extension and --disable-extensions-except flags.
- *
- * Usage: on_Crawl__30_chrome_launch.bg.js --crawl-id=<uuid> --source-url=<url>
- * Output: Writes to current directory (executor creates chrome/ dir):
- *   - cdp_url.txt: WebSocket URL for CDP connection
- *   - chrome.pid: Chromium process ID (for cleanup)
- *   - port.txt: Debug port number
- *   - extensions.json: Loaded extensions metadata
- *
- * Environment variables:
- *     NODE_MODULES_DIR: Path to node_modules directory for module resolution
- *     CHROME_BINARY: Path to Chromium binary (falls back to auto-detection)
- *     CHROME_RESOLUTION: Page resolution (default: 1440,2000)
- *     CHROME_HEADLESS: Run in headless mode (default: true)
- *     CHROME_CHECK_SSL_VALIDITY: Whether to check SSL certificates (default: true)
- *     CHROME_EXTENSIONS_DIR: Directory containing Chrome extensions
- */
-
-// Add NODE_MODULES_DIR to module resolution paths if set
-if (process.env.NODE_MODULES_DIR) {
-    module.paths.unshift(process.env.NODE_MODULES_DIR);
-}
-
-const fs = require('fs');
-const path = require('path');
-const puppeteer = require('puppeteer-core');
-const {
-    findChromium,
-    launchChromium,
-    killChrome,
-    getEnv,
-    writePidWithMtime,
-    getExtensionsDir,
-} = require('./chrome_utils.js');
-
-// Extractor metadata
-const PLUGIN_NAME = 'chrome_launch';
-const OUTPUT_DIR = '.';
-
-// Global state for cleanup
-let chromePid = null;
-let browserInstance = null;
-
-// Parse command line arguments
-function parseArgs() {
-    const args = {};
-    process.argv.slice(2).forEach((arg) => {
-        if (arg.startsWith('--')) {
-            const [key, ...valueParts] = arg.slice(2).split('=');
-            args[key.replace(/-/g, '_')] = valueParts.join('=') || true;
-        }
-    });
-    return args;
-}
-
-// Cleanup handler for SIGTERM
-async function cleanup() {
-    console.error('[*] Cleaning up Chrome session...');
-
-    // Try graceful browser close first
-    if (browserInstance) {
-        try {
-            console.error('[*] Closing browser gracefully...');
-            await browserInstance.close();
-            browserInstance = null;
-            console.error('[+] Browser closed gracefully');
-        } catch (e) {
-            console.error(`[!] Graceful close failed: ${e.message}`);
-        }
-    }
-
-    // Kill Chrome process
-    if (chromePid) {
-        await killChrome(chromePid, OUTPUT_DIR);
-    }
-
-    process.exit(0);
-}
-
-// Register signal handlers
-process.on('SIGTERM', cleanup);
-process.on('SIGINT', cleanup);
-
-async function main() {
-    const args = parseArgs();
-    const crawlId = args.crawl_id;
-
-    try {
-        const binary = findChromium();
-        if (!binary) {
-            console.error('ERROR: Chromium binary not found');
-            console.error('DEPENDENCY_NEEDED=chromium');
-            console.error('BIN_PROVIDERS=puppeteer,env,playwright,apt,brew');
-            console.error('INSTALL_HINT=npx @puppeteer/browsers install chromium@latest');
-            process.exit(1);
-        }
-
-        // Get Chromium version
-        let version = '';
-        try {
-            const { execSync } = require('child_process');
-            version = execSync(`"${binary}" --version`, { encoding: 'utf8', timeout: 5000 })
-                .trim()
-                .slice(0, 64);
-        } catch (e) {}
-
-        console.error(`[*] Using browser: ${binary}`);
-        if (version) console.error(`[*] Version: ${version}`);
-
-        // Load installed extensions
-        const extensionsDir = getExtensionsDir();
-        const userDataDir = getEnv('CHROME_USER_DATA_DIR');
-
-        if (userDataDir) {
-            console.error(`[*] Using user data dir: ${userDataDir}`);
-        }
-
-        const installedExtensions = [];
-        const extensionPaths = [];
-        if (fs.existsSync(extensionsDir)) {
-            const files = fs.readdirSync(extensionsDir);
-            for (const file of files) {
-                if (file.endsWith('.extension.json')) {
-                    try {
-                        const extPath = path.join(extensionsDir, file);
-                        const extData = JSON.parse(fs.readFileSync(extPath, 'utf-8'));
-                        if (extData.unpacked_path && fs.existsSync(extData.unpacked_path)) {
-                            installedExtensions.push(extData);
-                            extensionPaths.push(extData.unpacked_path);
-                            console.error(`[*] Loading extension: ${extData.name || file}`);
-                        }
-                    } catch (e) {
-                        console.warn(`[!] Skipping invalid extension cache: ${file}`);
-                    }
-                }
-            }
-        }
-
-        if (installedExtensions.length > 0) {
-            console.error(`[+] Found ${installedExtensions.length} extension(s) to load`);
-        }
-
-        // Note: PID file is written by run_hook() with hook-specific name
-        // Snapshot.cleanup() kills all *.pid processes when done
-        if (!fs.existsSync(OUTPUT_DIR)) {
-            fs.mkdirSync(OUTPUT_DIR, { recursive: true });
-        }
-
-        // Launch Chromium using consolidated function
-        // userDataDir is derived from ACTIVE_PERSONA by get_config() if not explicitly set
-        const result = await launchChromium({
-            binary,
-            outputDir: OUTPUT_DIR,
-            userDataDir,
-            extensionPaths,
-        });
-
-        if (!result.success) {
-            console.error(`ERROR: ${result.error}`);
-            process.exit(1);
-        }
-
-        chromePid = result.pid;
-        const cdpUrl = result.cdpUrl;
-
-        // Connect puppeteer for extension verification
-        console.error(`[*] Connecting puppeteer to CDP...`);
-        const browser = await puppeteer.connect({
-            browserWSEndpoint: cdpUrl,
-            defaultViewport: null,
-        });
-        browserInstance = browser;
-
-        // Get actual extension IDs from chrome://extensions page
-        if (extensionPaths.length > 0) {
-            await new Promise(r => setTimeout(r, 2000));
-
-            try {
-                const extPage = await browser.newPage();
-                await extPage.goto('chrome://extensions', { waitUntil: 'domcontentloaded', timeout: 10000 });
-                await new Promise(r => setTimeout(r, 2000));
-
-                // Parse extension info from the page
-                const extensionsFromPage = await extPage.evaluate(() => {
-                    const extensions = [];
-                    // Extensions manager uses shadow DOM
-                    const manager = document.querySelector('extensions-manager');
-                    if (!manager || !manager.shadowRoot) return extensions;
-
-                    const itemList = manager.shadowRoot.querySelector('extensions-item-list');
-                    if (!itemList || !itemList.shadowRoot) return extensions;
-
-                    const items = itemList.shadowRoot.querySelectorAll('extensions-item');
-                    for (const item of items) {
-                        const id = item.getAttribute('id');
-                        const nameEl = item.shadowRoot?.querySelector('#name');
-                        const name = nameEl?.textContent?.trim() || '';
-                        if (id && name) {
-                            extensions.push({ id, name });
-                        }
-                    }
-                    return extensions;
-                });
-
-                console.error(`[*] Found ${extensionsFromPage.length} extension(s) on chrome://extensions`);
-                for (const e of extensionsFromPage) {
-                    console.error(`    - ${e.id}: "${e.name}"`);
-                }
-
-                // Match extensions by name (strict matching)
-                for (const ext of installedExtensions) {
-                    // Read the extension's manifest to get its display name
-                    const manifestPath = path.join(ext.unpacked_path, 'manifest.json');
-                    if (fs.existsSync(manifestPath)) {
-                        const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8'));
-                        let manifestName = manifest.name || '';
-
-                        // Resolve message placeholder (e.g., __MSG_extName__)
-                        if (manifestName.startsWith('__MSG_') && manifestName.endsWith('__')) {
-                            const msgKey = manifestName.slice(6, -2); // Extract key from __MSG_key__
-                            const defaultLocale = manifest.default_locale || 'en';
-                            const messagesPath = path.join(ext.unpacked_path, '_locales', defaultLocale, 'messages.json');
-                            if (fs.existsSync(messagesPath)) {
-                                try {
-                                    const messages = JSON.parse(fs.readFileSync(messagesPath, 'utf-8'));
-                                    if (messages[msgKey] && messages[msgKey].message) {
-                                        manifestName = messages[msgKey].message;
-                                    }
-                                } catch (e) {
-                                    console.error(`[!] Failed to read messages.json: ${e.message}`);
-                                }
-                            }
-                        }
-
-                        console.error(`[*] Looking for match: ext.name="${ext.name}" manifest.name="${manifestName}"`);
-
-                        // Find matching extension from page by exact name match first
-                        let match = extensionsFromPage.find(e => e.name === manifestName);
-
-                        // If no exact match, try case-insensitive exact match
-                        if (!match) {
-                            match = extensionsFromPage.find(e =>
-                                e.name.toLowerCase() === manifestName.toLowerCase()
-                            );
-                        }
-
-                        if (match) {
-                            ext.id = match.id;
-                            console.error(`[+] Matched extension: ${ext.name} (${manifestName}) -> ${match.id}`);
-                        } else {
-                            console.error(`[!] No match found for: ${ext.name} (${manifestName})`);
-                        }
-                    }
-                }
-
-                await extPage.close();
-            } catch (e) {
-                console.error(`[!] Failed to get extensions from chrome://extensions: ${e.message}`);
-            }
-
-            // Fallback: check browser targets
-            const targets = browser.targets();
-            const builtinIds = [
-                'nkeimhogjdpnpccoofpliimaahmaaome',
-                'fignfifoniblkonapihmkfakmlgkbkcf',
-                'ahfgeienlihckogmohjhadlkjgocpleb',
-                'mhjfbmdgcfjbbpaeojofohoefgiehjai',
-            ];
-            const customExtTargets = targets.filter(t => {
-                const url = t.url();
-                if (!url.startsWith('chrome-extension://')) return false;
-                const extId = url.split('://')[1].split('/')[0];
-                return !builtinIds.includes(extId);
-            });
-
-            console.error(`[+] Found ${customExtTargets.length} custom extension target(s)`);
-
-            for (const target of customExtTargets) {
-                const url = target.url();
-                const extId = url.split('://')[1].split('/')[0];
-                console.error(`[+] Extension target: ${extId} (${target.type()})`);
-            }
-
-            if (customExtTargets.length === 0 && extensionPaths.length > 0) {
-                console.error(`[!] Warning: No custom extensions detected. Extension loading may have failed.`);
-                console.error(`[!] Make sure you are using Chromium, not Chrome (Chrome 137+ removed --load-extension support)`);
-            }
-        }
-
-        // Write extensions metadata with actual IDs
-        if (installedExtensions.length > 0) {
-            fs.writeFileSync(
-                path.join(OUTPUT_DIR, 'extensions.json'),
-                JSON.stringify(installedExtensions, null, 2)
-            );
-        }
-
-        console.error(`[+] Chromium session started for crawl ${crawlId}`);
-        console.error(`[+] CDP URL: ${cdpUrl}`);
-        console.error(`[+] PID: ${chromePid}`);
-
-        // Stay alive to handle cleanup on SIGTERM
-        console.log('[*] Chromium launch hook staying alive to handle cleanup...');
-        setInterval(() => {}, 1000000);
-
-    } catch (e) {
-        console.error(`ERROR: ${e.name}: ${e.message}`);
-        process.exit(1);
-    }
-}
-
-main().catch((e) => {
-    console.error(`Fatal error: ${e.message}`);
-    process.exit(1);
-});