finish migrating almost all config to new system

archivebox/plugins_extractor/curl/apps.py

@@ -1,10 +1,10 @@
 __package__ = 'plugins_extractor.curl'
 
-from typing import List, Optional, Dict
+from typing import List, Optional
 from pathlib import Path
 
 from pydantic import InstanceOf, Field
-from pydantic_pkgr import BinProvider, BinName, bin_abspath, BinProviderName, ProviderLookupDict
+from pydantic_pkgr import BinProvider, BinName
 
 from abx.archivebox.base_plugin import BasePlugin, BaseHook
 from abx.archivebox.base_configset import BaseConfigSet
@@ -12,15 +12,26 @@ from abx.archivebox.base_binary import BaseBinary, env, apt, brew
 # from abx.archivebox.base_extractor import BaseExtractor, ExtractorName
 
 from archivebox.config import ARCHIVING_CONFIG
-
+from archivebox.plugins_extractor.favicon.apps import FAVICON_CONFIG
+from archivebox.plugins_extractor.archivedotorg.apps import ARCHIVEDOTORG_CONFIG
 
 class CurlConfig(BaseConfigSet):
-
-    SAVE_CURL: bool = True
     
-    # USE_CURL: bool = Field(default=lambda c: c.SAVE_HEADERS or c.SAVE_FAVICON)
+    SAVE_TITLE: bool = Field(default=True)
+    SAVE_HEADERS: bool = Field(default=True)
+    USE_CURL: bool = Field(default=lambda c: 
+        ARCHIVEDOTORG_CONFIG.SAVE_ARCHIVE_DOT_ORG
+        or FAVICON_CONFIG.SAVE_FAVICON
+        or c.SAVE_HEADERS
+        or c.SAVE_TITLE
+    )
     
     CURL_BINARY: str = Field(default='curl')
+    CURL_ARGS: List[str] = [
+        '--silent',
+        '--location',
+        '--compressed',
+    ]
     CURL_EXTRA_ARGS: List[str] = []
     
     CURL_TIMEOUT: int =  Field(default=lambda: ARCHIVING_CONFIG.TIMEOUT)
@@ -35,12 +46,6 @@ CURL_CONFIG = CurlConfig()
 class CurlBinary(BaseBinary):
     name: BinName = CURL_CONFIG.CURL_BINARY
     binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
-    
-    provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
-        brew.name: {
-            'abspath': lambda: bin_abspath(CURL_CONFIG.CURL_BINARY, PATH=f'/opt/homebrew/opt/curl/bin:{brew.PATH}'),
-        },
-    }
 
 CURL_BINARY = CurlBinary()
 

archivebox/plugins_extractor/wget/apps.py

@@ -1,13 +1,13 @@
 __package__ = 'plugins_extractor.wget'
 
 import sys
-from typing import List, Optional, Dict
+from typing import List, Optional
 from pathlib import Path
 from subprocess import run, DEVNULL
 
 from rich import print
 from pydantic import InstanceOf, Field, model_validator
-from pydantic_pkgr import BinProvider, BinName, bin_abspath, BinProviderName, ProviderLookupDict
+from pydantic_pkgr import BinProvider, BinName
 
 from abx.archivebox.base_plugin import BasePlugin, BaseHook
 from abx.archivebox.base_configset import BaseConfigSet
@@ -80,12 +80,6 @@ WGET_CONFIG = WgetConfig()
 class WgetBinary(BaseBinary):
     name: BinName = WGET_CONFIG.WGET_BINARY
     binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
-    
-    provider_overrides: Dict[BinProviderName, ProviderLookupDict] = {
-        brew.name: {
-            'abspath': lambda: bin_abspath(WGET_CONFIG.WGET_BINARY, PATH=f'/opt/homebrew/opt/wget/bin:{brew.PATH}'),
-        },
-    }
 
 WGET_BINARY = WgetBinary()