alex/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-04-05 15:27:53 +10:00
Code Issues Packages Projects Releases Wiki Activity

move abx plugins inside vendor dir

Browse Source
This commit is contained in:
Nick Sweeting
2024-10-28 04:07:35 -07:00
parent 5d9a32c364
commit b3c1cb716e
242 changed files with 2153 additions and 2700 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
archivebox/__init__.py
View File

@@ -15,7 +15,7 @@ import os
import sys
from pathlib import Path
from typing import cast
ASCII_LOGO = """
█████╗ ██████╗ ██████╗██╗ ██╗██╗██╗ ██╗███████╗ ██████╗ ██████╗ ██╗ ██╗
██╔══██╗██╔══██╗██╔════╝██║ ██║██║██║ ██║██╔════╝ ██╔══██╗██╔═══██╗╚██╗██╔╝
@@ -52,6 +52,50 @@ load_vendored_libs()
# print('DONE LOADING VENDORED LIBRARIES')
import abx # noqa
import abx_spec_archivebox # noqa
import abx_spec_config # noqa
import abx_spec_pydantic_pkgr # noqa
import abx_spec_django # noqa
import abx_spec_searchbackend # noqa
abx.pm.add_hookspecs(abx_spec_config.PLUGIN_SPEC)
abx.pm.register(abx_spec_config.PLUGIN_SPEC())
abx.pm.add_hookspecs(abx_spec_pydantic_pkgr.PLUGIN_SPEC)
abx.pm.register(abx_spec_pydantic_pkgr.PLUGIN_SPEC())
abx.pm.add_hookspecs(abx_spec_django.PLUGIN_SPEC)
abx.pm.register(abx_spec_django.PLUGIN_SPEC())
abx.pm.add_hookspecs(abx_spec_searchbackend.PLUGIN_SPEC)
abx.pm.register(abx_spec_searchbackend.PLUGIN_SPEC())
abx.pm = cast(abx.ABXPluginManager[abx_spec_archivebox.ArchiveBoxPluginSpec], abx.pm)
pm = abx.pm
# Load all installed ABX-compatible plugins
ABX_ECOSYSTEM_PLUGINS = abx.get_pip_installed_plugins(group='abx')
# Load all ArchiveBox-specific plugins
ARCHIVEBOX_BUILTIN_PLUGINS = {
'config': PACKAGE_DIR / 'config',
'core': PACKAGE_DIR / 'core',
# 'search': PACKAGE_DIR / 'search',
# 'core': PACKAGE_DIR / 'core',
}
# Load all user-defined ArchiveBox plugins
USER_PLUGINS = abx.find_plugins_in_dir(Path(os.getcwd()) / 'user_plugins')
# Merge all plugins together
ALL_PLUGINS = {**ABX_ECOSYSTEM_PLUGINS, **ARCHIVEBOX_BUILTIN_PLUGINS, **USER_PLUGINS}
# Load ArchiveBox plugins
LOADED_PLUGINS = abx.load_plugins(ALL_PLUGINS)
from .config.constants import CONSTANTS # noqa
from .config.paths import PACKAGE_DIR, DATA_DIR, ARCHIVE_DIR # noqa
from .config.version import VERSION # noqa

30
archivebox/config/__init__.py
View File

@@ -1,4 +1,5 @@
__package__ = 'archivebox.config'
__package__ = 'config'
__order__ = 200
from .paths import (
PACKAGE_DIR, # noqa
@@ -9,30 +10,3 @@ from .constants import CONSTANTS, CONSTANTS_CONFIG, PACKAGE_DIR, DATA_DIR, ARCHI
from .version import VERSION # noqa
import abx
# @abx.hookimpl
# def get_INSTALLED_APPS():
# return ['config']
@abx.hookimpl
def get_CONFIG():
from .common import (
SHELL_CONFIG,
STORAGE_CONFIG,
GENERAL_CONFIG,
SERVER_CONFIG,
ARCHIVING_CONFIG,
SEARCH_BACKEND_CONFIG,
)
return {
'SHELL_CONFIG': SHELL_CONFIG,
'STORAGE_CONFIG': STORAGE_CONFIG,
'GENERAL_CONFIG': GENERAL_CONFIG,
'SERVER_CONFIG': SERVER_CONFIG,
'ARCHIVING_CONFIG': ARCHIVING_CONFIG,
'SEARCHBACKEND_CONFIG': SEARCH_BACKEND_CONFIG,
}

18
archivebox/config/configfile.py → archivebox/config/collection.py
View File

@@ -9,6 +9,8 @@ from configparser import ConfigParser
from benedict import benedict
import archivebox
from archivebox.config.constants import CONSTANTS
from archivebox.misc.logging import stderr
@@ -16,9 +18,9 @@ from archivebox.misc.logging import stderr
def get_real_name(key: str) -> str:
"""get the up-to-date canonical name for a given old alias or current key"""
from django.conf import settings
CONFIGS = archivebox.pm.hook.get_CONFIGS()
for section in settings.CONFIGS.values():
for section in CONFIGS.values():
try:
return section.aliases[key]
except KeyError:
@@ -115,17 +117,15 @@ def load_config_file() -> Optional[benedict]:
def section_for_key(key: str) -> Any:
from django.conf import settings
for config_section in settings.CONFIGS.values():
for config_section in archivebox.pm.hook.get_CONFIGS().values():
if hasattr(config_section, key):
return config_section
return None
raise ValueError(f'No config section found for key: {key}')
def write_config_file(config: Dict[str, str]) -> benedict:
"""load the ini-formatted config file from DATA_DIR/Archivebox.conf"""
import abx.archivebox.reads
from archivebox.misc.system import atomic_write
CONFIG_HEADER = (
@@ -175,7 +175,7 @@ def write_config_file(config: Dict[str, str]) -> benedict:
updated_config = {}
try:
# validate the updated_config by attempting to re-parse it
updated_config = {**load_all_config(), **abx.archivebox.reads.get_FLAT_CONFIG()}
updated_config = {**load_all_config(), **archivebox.pm.hook.get_FLAT_CONFIG()}
except BaseException: # lgtm [py/catch-base-exception]
# something went horribly wrong, revert to the previous version
with open(f'{config_path}.bak', 'r', encoding='utf-8') as old:
@@ -233,11 +233,11 @@ def load_config(defaults: Dict[str, Any],
return benedict(extended_config)
def load_all_config():
import abx.archivebox.reads
import abx
flat_config = benedict()
for config_section in abx.archivebox.reads.get_CONFIGS().values():
for config_section in abx.pm.hook.get_CONFIGS().values():
config_section.__init__()
flat_config.update(config_section.model_dump())

11
archivebox/config/common.py
View File

@@ -7,10 +7,10 @@ from typing import Dict, Optional, List
from pathlib import Path
from rich import print
from pydantic import Field, field_validator, computed_field
from pydantic import Field, field_validator
from django.utils.crypto import get_random_string
from abx.archivebox.base_configset import BaseConfigSet
from abx_spec_config.base_configset import BaseConfigSet
from .constants import CONSTANTS
from .version import get_COMMIT_HASH, get_BUILD_TIME
@@ -31,22 +31,19 @@ class ShellConfig(BaseConfigSet):
ANSI: Dict[str, str] = Field(default=lambda c: CONSTANTS.DEFAULT_CLI_COLORS if c.USE_COLOR else CONSTANTS.DISABLED_CLI_COLORS)
VERSIONS_AVAILABLE: bool = False # .check_for_update.get_versions_available_on_github(c)},
CAN_UPGRADE: bool = False # .check_for_update.can_upgrade(c)},
# VERSIONS_AVAILABLE: bool = False # .check_for_update.get_versions_available_on_github(c)},
# CAN_UPGRADE: bool = False # .check_for_update.can_upgrade(c)},
@computed_field
@property
def TERM_WIDTH(self) -> int:
if not self.IS_TTY:
return 200
return shutil.get_terminal_size((140, 10)).columns
@computed_field
@property
def COMMIT_HASH(self) -> Optional[str]:
return get_COMMIT_HASH()
@computed_field
@property
def BUILD_TIME(self) -> str:
return get_BUILD_TIME()

2
archivebox/config/django.py
View File

@@ -97,7 +97,7 @@ def setup_django(check_db=False, in_memory_db=False) -> None:
except Exception as e:
bump_startup_progress_bar(advance=1000)
is_using_meta_cmd = any(ignored_subcommand in sys.argv for ignored_subcommand in ('help', 'version', '--help', '--version', 'init'))
is_using_meta_cmd = any(ignored_subcommand in sys.argv for ignored_subcommand in ('help', 'version', '--help', '--version'))
if not is_using_meta_cmd:
# show error message to user only if they're not running a meta command / just trying to get help
STDERR.print()

28
archivebox/config/views.py
View File

@@ -14,8 +14,8 @@ from django.utils.html import format_html, mark_safe
from admin_data_views.typing import TableContext, ItemContext
from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink
import abx.archivebox.reads
import abx
import archivebox
from archivebox.config import CONSTANTS
from archivebox.misc.util import parse_date
@@ -65,7 +65,7 @@ def obj_to_yaml(obj: Any, indent: int=0) -> str:
@render_with_table_view
def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
FLAT_CONFIG = archivebox.pm.hook.get_FLAT_CONFIG()
assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
rows = {
@@ -81,12 +81,11 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
relevant_configs = {
key: val
for key, val in settings.FLAT_CONFIG.items()
for key, val in FLAT_CONFIG.items()
if '_BINARY' in key or '_VERSION' in key
}
for plugin_id, plugin in abx.archivebox.reads.get_PLUGINS().items():
plugin = abx.archivebox.reads.get_PLUGIN(plugin_id)
for plugin_id, plugin in abx.get_all_plugins().items():
if not plugin.hooks.get('get_BINARIES'):
continue
@@ -131,17 +130,16 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
@render_with_item_view
def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
assert request.user and request.user.is_superuser, 'Must be a superuser to view configuration settings.'
binary = None
plugin = None
for plugin_id in abx.archivebox.reads.get_PLUGINS().keys():
loaded_plugin = abx.archivebox.reads.get_PLUGIN(plugin_id)
for plugin_id, plugin in abx.get_all_plugins().items():
try:
for loaded_binary in loaded_plugin.hooks.get_BINARIES().values():
for loaded_binary in plugin['hooks'].get_BINARIES().values():
if loaded_binary.name == key:
binary = loaded_binary
plugin = loaded_plugin
plugin = plugin
# break # last write wins
except Exception as e:
print(e)
@@ -161,7 +159,7 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
"name": binary.name,
"description": binary.abspath,
"fields": {
'plugin': plugin.package,
'plugin': plugin['package'],
'binprovider': binary.loaded_binprovider,
'abspath': binary.loaded_abspath,
'version': binary.loaded_version,
@@ -215,9 +213,7 @@ def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
return color
return 'black'
for plugin_id in settings.PLUGINS.keys():
plugin = abx.archivebox.reads.get_PLUGIN(plugin_id)
for plugin_id, plugin in abx.get_all_plugins().items():
plugin.hooks.get_BINPROVIDERS = plugin.hooks.get('get_BINPROVIDERS', lambda: {})
plugin.hooks.get_BINARIES = plugin.hooks.get('get_BINARIES', lambda: {})
plugin.hooks.get_CONFIG = plugin.hooks.get('get_CONFIG', lambda: {})
@@ -263,7 +259,7 @@ def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
assert plugin_id, f'Could not find a plugin matching the specified name: {key}'
plugin = abx.archivebox.reads.get_PLUGIN(plugin_id)
plugin = abx.get_plugin(plugin_id)
return ItemContext(
slug=key,

29
archivebox/core/__init__.py
View File

@@ -1,2 +1,31 @@
__package__ = 'archivebox.core'
import abx
@abx.hookimpl
def register_admin(admin_site):
"""Register the core.models views (Snapshot, ArchiveResult, Tag, etc.) with the admin site"""
from core.admin import register_admin
register_admin(admin_site)
@abx.hookimpl
def get_CONFIG():
from archivebox.config.common import (
SHELL_CONFIG,
STORAGE_CONFIG,
GENERAL_CONFIG,
SERVER_CONFIG,
ARCHIVING_CONFIG,
SEARCH_BACKEND_CONFIG,
)
return {
'SHELL_CONFIG': SHELL_CONFIG,
'STORAGE_CONFIG': STORAGE_CONFIG,
'GENERAL_CONFIG': GENERAL_CONFIG,
'SERVER_CONFIG': SERVER_CONFIG,
'ARCHIVING_CONFIG': ARCHIVING_CONFIG,
'SEARCHBACKEND_CONFIG': SEARCH_BACKEND_CONFIG,
}

4
archivebox/core/admin_site.py
View File

@@ -2,7 +2,7 @@ __package__ = 'archivebox.core'
from django.contrib import admin
import abx.django.use
import archivebox
class ArchiveBoxAdmin(admin.AdminSite):
site_header = 'ArchiveBox'
@@ -37,6 +37,6 @@ def register_admin_site():
sites.site = archivebox_admin
# register all plugins admin classes
abx.django.use.register_admin(archivebox_admin)
archivebox.pm.hook.register_admin(admin_site=archivebox_admin)
return archivebox_admin

13
archivebox/core/apps.py
View File

@@ -2,7 +2,7 @@ __package__ = 'archivebox.core'
from django.apps import AppConfig
import abx
import archivebox
class CoreConfig(AppConfig):
@@ -10,16 +10,11 @@ class CoreConfig(AppConfig):
def ready(self):
"""Register the archivebox.core.admin_site as the main django admin site"""
from django.conf import settings
archivebox.pm.hook.ready(settings=settings)
from core.admin_site import register_admin_site
register_admin_site()
abx.pm.hook.ready()
@abx.hookimpl
def register_admin(admin_site):
"""Register the core.models views (Snapshot, ArchiveResult, Tag, etc.) with the admin site"""
from core.admin import register_admin
register_admin(admin_site)

25
archivebox/core/settings.py
View File

@@ -9,10 +9,12 @@ from pathlib import Path
from django.utils.crypto import get_random_string
import abx
import archivebox
from archivebox.config import DATA_DIR, PACKAGE_DIR, ARCHIVE_DIR, CONSTANTS
from archivebox.config import DATA_DIR, PACKAGE_DIR, ARCHIVE_DIR, CONSTANTS # noqa
from archivebox.config.common import SHELL_CONFIG, SERVER_CONFIG # noqa
IS_MIGRATING = 'makemigrations' in sys.argv[:3] or 'migrate' in sys.argv[:3]
IS_TESTING = 'test' in sys.argv[:3] or 'PYTEST_CURRENT_TEST' in os.environ
IS_SHELL = 'shell' in sys.argv[:3] or 'shell_plus' in sys.argv[:3]
@@ -22,24 +24,8 @@ IS_GETTING_VERSION_OR_HELP = 'version' in sys.argv or 'help' in sys.argv or '--v
### ArchiveBox Plugin Settings
################################################################################
PLUGIN_HOOKSPECS = [
'abx_spec_django',
'abx_spec_pydantic_pkgr',
'abx_spec_config',
'abx_spec_archivebox',
]
abx.register_hookspecs(PLUGIN_HOOKSPECS)
SYSTEM_PLUGINS = abx.get_pip_installed_plugins(group='abx')
USER_PLUGINS = abx.find_plugins_in_dir(DATA_DIR / 'user_plugins')
ALL_PLUGINS = {**SYSTEM_PLUGINS, **USER_PLUGINS}
# Load ArchiveBox plugins
abx.load_plugins(ALL_PLUGINS)
# # Load ArchiveBox config from plugins
ALL_PLUGINS = archivebox.ALL_PLUGINS
LOADED_PLUGINS = archivebox.LOADED_PLUGINS
################################################################################
### Django Core Settings
@@ -101,6 +87,7 @@ INSTALLED_APPS = [
MIDDLEWARE = [
'core.middleware.TimezoneMiddleware',
'django.middleware.security.SecurityMiddleware',

5
archivebox/core/settings_logging.py
View File

@@ -163,11 +163,6 @@ SETTINGS_LOGGING = {
"level": "DEBUG",
"propagate": False,
},
"plugins_extractor": {
"handlers": ["default", "logfile"],
"level": "DEBUG",
"propagate": False,
},
"httpx": {
"handlers": ["outbound_webhooks"],
"level": "INFO",

38
archivebox/core/views.py
View File

@@ -21,6 +21,7 @@ from django.utils.decorators import method_decorator
from admin_data_views.typing import TableContext, ItemContext
from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink
import archivebox
from core.models import Snapshot
from core.forms import AddLinkForm
@@ -32,9 +33,8 @@ from archivebox.config.common import SHELL_CONFIG, SERVER_CONFIG
from archivebox.misc.util import base_url, htmlencode, ts_to_date_str
from archivebox.misc.serve_static import serve_static_with_byterange_support
from ..plugins_extractor.archivedotorg.config import ARCHIVEDOTORG_CONFIG
from ..logging_util import printable_filesize
from ..search import query_search_index
from archivebox.logging_util import printable_filesize
from archivebox.search import query_search_index
class HomepageView(View):
@@ -154,7 +154,7 @@ class SnapshotView(View):
'status_color': 'success' if link.is_archived else 'danger',
'oldest_archive_date': ts_to_date_str(link.oldest_archive_date),
'warc_path': warc_path,
'SAVE_ARCHIVE_DOT_ORG': ARCHIVEDOTORG_CONFIG.SAVE_ARCHIVE_DOT_ORG,
'SAVE_ARCHIVE_DOT_ORG': archivebox.pm.hook.get_FLAT_CONFIG().SAVE_ARCHIVE_DOT_ORG,
'PREVIEW_ORIGINALS': SERVER_CONFIG.PREVIEW_ORIGINALS,
'archiveresults': sorted(archiveresults.values(), key=lambda r: all_types.index(r['name']) if r['name'] in all_types else -r['size']),
'best_result': best_result,
@@ -500,21 +500,25 @@ class HealthCheckView(View):
def find_config_section(key: str) -> str:
CONFIGS = archivebox.pm.hook.get_CONFIGS()
if key in CONSTANTS_CONFIG:
return 'CONSTANT'
matching_sections = [
section_id for section_id, section in settings.CONFIGS.items() if key in section.model_fields
section_id for section_id, section in CONFIGS.items() if key in section.model_fields
]
section = matching_sections[0] if matching_sections else 'DYNAMIC'
return section
def find_config_default(key: str) -> str:
CONFIGS = archivebox.pm.hook.get_CONFIGS()
if key in CONSTANTS_CONFIG:
return str(CONSTANTS_CONFIG[key])
default_val = None
for config in settings.CONFIGS.values():
for config in CONFIGS.values():
if key in config.model_fields:
default_val = config.model_fields[key].default
break
@@ -530,7 +534,9 @@ def find_config_default(key: str) -> str:
return default_val
def find_config_type(key: str) -> str:
for config in settings.CONFIGS.values():
CONFIGS = archivebox.pm.hook.get_CONFIGS()
for config in CONFIGS.values():
if hasattr(config, key):
type_hints = get_type_hints(config)
try:
@@ -547,7 +553,8 @@ def key_is_safe(key: str) -> bool:
@render_with_table_view
def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
CONFIGS = archivebox.pm.hook.get_CONFIGS()
assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
rows = {
@@ -560,7 +567,7 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
# "Aliases": [],
}
for section_id, section in reversed(list(settings.CONFIGS.items())):
for section_id, section in reversed(list(CONFIGS.items())):
for key, field in section.model_fields.items():
rows['Section'].append(section_id) # section.replace('_', ' ').title().replace(' Config', '')
rows['Key'].append(ItemLink(key, key=key))
@@ -570,7 +577,6 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
# rows['Documentation'].append(mark_safe(f'Wiki: <a href="https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#{key.lower()}">{key}</a>'))
# rows['Aliases'].append(', '.join(find_config_aliases(key)))
section = 'CONSTANT'
for key in CONSTANTS_CONFIG.keys():
rows['Section'].append(section) # section.replace('_', ' ').title().replace(' Config', '')
@@ -589,7 +595,9 @@ def live_config_list_view(request: HttpRequest, **kwargs) -> TableContext:
@render_with_item_view
def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
CONFIGS = archivebox.pm.hook.get_CONFIGS()
FLAT_CONFIG = archivebox.pm.hook.get_FLAT_CONFIG()
assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
# aliases = USER_CONFIG.get(key, {}).get("aliases", [])
@@ -597,7 +605,7 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
if key in CONSTANTS_CONFIG:
section_header = mark_safe(f'[CONSTANTS] &nbsp; <b><code style="color: lightgray">{key}</code></b> &nbsp; <small>(read-only, hardcoded by ArchiveBox)</small>')
elif key in settings.FLAT_CONFIG:
elif key in FLAT_CONFIG:
section_header = mark_safe(f'data / ArchiveBox.conf &nbsp; [{find_config_section(key)}] &nbsp; <b><code style="color: lightgray">{key}</code></b>')
else:
section_header = mark_safe(f'[DYNAMIC CONFIG] &nbsp; <b><code style="color: lightgray">{key}</code></b> &nbsp; <small>(read-only, calculated at runtime)</small>')
@@ -613,7 +621,7 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
"fields": {
'Key': key,
'Type': find_config_type(key),
'Value': settings.FLAT_CONFIG.get(key, settings.CONFIGS.get(key, None)) if key_is_safe(key) else '********',
'Value': FLAT_CONFIG.get(key, CONFIGS.get(key, None)) if key_is_safe(key) else '********',
},
"help_texts": {
'Key': mark_safe(f'''
@@ -635,13 +643,13 @@ def live_config_value_view(request: HttpRequest, key: str, **kwargs) -> ItemCont
<code>{find_config_default(key) or '↗️ See in ArchiveBox source code...'}</code>
</a>
<br/><br/>
<p style="display: {"block" if key in settings.FLAT_CONFIG else "none"}">
<p style="display: {"block" if key in FLAT_CONFIG else "none"}">
<i>To change this value, edit <code>data/ArchiveBox.conf</code> or run:</i>
<br/><br/>
<code>archivebox config --set {key}="{
val.strip("'")
if (val := find_config_default(key)) else
(repr(settings.FLAT_CONFIG[key] if key_is_safe(key) else '********')).strip("'")
(repr(FLAT_CONFIG[key] if key_is_safe(key) else '********')).strip("'")
}"</code>
</p>
'''),

46
archivebox/extractors/__init__.py
View File

@@ -27,43 +27,29 @@ from ..logging_util import (
log_archive_method_finished,
)
from .title import should_save_title, save_title
from .favicon import should_save_favicon, save_favicon
from .wget import should_save_wget, save_wget
from .singlefile import should_save_singlefile, save_singlefile
from .readability import should_save_readability, save_readability
from .mercury import should_save_mercury, save_mercury
from .htmltotext import should_save_htmltotext, save_htmltotext
from .pdf import should_save_pdf, save_pdf
from .screenshot import should_save_screenshot, save_screenshot
from .dom import should_save_dom, save_dom
from .git import should_save_git, save_git
from .media import should_save_media, save_media
from .archive_org import should_save_archive_dot_org, save_archive_dot_org
from .headers import should_save_headers, save_headers
ShouldSaveFunction = Callable[[Link, Optional[Path], Optional[bool]], bool]
SaveFunction = Callable[[Link, Optional[Path], int], ArchiveResult]
ArchiveMethodEntry = tuple[str, ShouldSaveFunction, SaveFunction]
def get_default_archive_methods() -> List[ArchiveMethodEntry]:
# TODO: move to abx.pm.hook.get_EXTRACTORS()
return [
('favicon', should_save_favicon, save_favicon),
('headers', should_save_headers, save_headers),
('singlefile', should_save_singlefile, save_singlefile),
('pdf', should_save_pdf, save_pdf),
('screenshot', should_save_screenshot, save_screenshot),
('dom', should_save_dom, save_dom),
('wget', should_save_wget, save_wget),
# keep title, readability, and htmltotext below wget and singlefile, as they depend on them
('title', should_save_title, save_title),
('readability', should_save_readability, save_readability),
('mercury', should_save_mercury, save_mercury),
('htmltotext', should_save_htmltotext, save_htmltotext),
('git', should_save_git, save_git),
('media', should_save_media, save_media),
('archive_org', should_save_archive_dot_org, save_archive_dot_org),
# ('favicon', should_save_favicon, save_favicon),
# ('headers', should_save_headers, save_headers),
# ('singlefile', should_save_singlefile, save_singlefile),
# ('pdf', should_save_pdf, save_pdf),
# ('screenshot', should_save_screenshot, save_screenshot),
# ('dom', should_save_dom, save_dom),
# ('wget', should_save_wget, save_wget),
# # keep title, readability, and htmltotext below wget and singlefile, as they depend on them
# ('title', should_save_title, save_title),
# ('readability', should_save_readability, save_readability),
# ('mercury', should_save_mercury, save_mercury),
# ('htmltotext', should_save_htmltotext, save_htmltotext),
# ('git', should_save_git, save_git),
# ('media', should_save_media, save_media),
# ('archive_org', should_save_archive_dot_org, save_archive_dot_org),
]
ARCHIVE_METHODS_INDEXING_PRECEDENCE = [

11
archivebox/index/html.py
View File

@@ -8,6 +8,8 @@ from typing import List, Optional, Iterator, Mapping
from django.utils.html import format_html, mark_safe # type: ignore
from django.core.cache import cache
import abx
from archivebox.misc.system import atomic_write
from archivebox.misc.util import (
enforce_types,
@@ -19,7 +21,6 @@ from archivebox.misc.util import (
from archivebox.config import CONSTANTS, DATA_DIR, VERSION
from archivebox.config.common import SERVER_CONFIG
from archivebox.config.version import get_COMMIT_HASH
from archivebox.plugins_extractor.archivedotorg.config import ARCHIVEDOTORG_CONFIG
from .schema import Link
from ..logging_util import printable_filesize
@@ -79,8 +80,10 @@ def write_html_link_details(link: Link, out_dir: Optional[str]=None) -> None:
@enforce_types
def link_details_template(link: Link) -> str:
from ..extractors.wget import wget_output_path
from abx_plugin_wget_extractor.wget import wget_output_path
SAVE_ARCHIVE_DOT_ORG = abx.pm.hook.get_FLAT_CONFIG().SAVE_ARCHIVE_DOT_ORG
link_info = link._asdict(extended=True)
@@ -102,7 +105,7 @@ def link_details_template(link: Link) -> str:
'status': 'archived' if link.is_archived else 'not yet archived',
'status_color': 'success' if link.is_archived else 'danger',
'oldest_archive_date': ts_to_date_str(link.oldest_archive_date),
'SAVE_ARCHIVE_DOT_ORG': ARCHIVEDOTORG_CONFIG.SAVE_ARCHIVE_DOT_ORG,
'SAVE_ARCHIVE_DOT_ORG': SAVE_ARCHIVE_DOT_ORG,
'PREVIEW_ORIGINALS': SERVER_CONFIG.PREVIEW_ORIGINALS,
})

4
archivebox/index/json.py
View File

@@ -8,7 +8,7 @@ from pathlib import Path
from datetime import datetime, timezone
from typing import List, Optional, Iterator, Any, Union
import abx.archivebox.reads
import abx
from archivebox.config import VERSION, DATA_DIR, CONSTANTS
from archivebox.config.common import SERVER_CONFIG, SHELL_CONFIG
@@ -33,7 +33,7 @@ def generate_json_index_from_links(links: List[Link], with_headers: bool):
'docs': 'https://github.com/ArchiveBox/ArchiveBox/wiki',
'source': 'https://github.com/ArchiveBox/ArchiveBox',
'issues': 'https://github.com/ArchiveBox/ArchiveBox/issues',
'dependencies': dict(abx.archivebox.reads.get_BINARIES()),
'dependencies': dict(abx.pm.hook.get_BINARIES()),
},
}

9
archivebox/index/schema.py
View File

@@ -17,9 +17,9 @@ from dataclasses import dataclass, asdict, field, fields
from django.utils.functional import cached_property
from archivebox.config import ARCHIVE_DIR, CONSTANTS
import abx
from plugins_extractor.favicon.config import FAVICON_CONFIG
from archivebox.config import ARCHIVE_DIR, CONSTANTS
from archivebox.misc.system import get_dir_size
from archivebox.misc.util import ts_to_date_str, parse_date
@@ -426,7 +426,10 @@ class Link:
def canonical_outputs(self) -> Dict[str, Optional[str]]:
"""predict the expected output paths that should be present after archiving"""
from ..extractors.wget import wget_output_path
from abx_plugin_wget.wget import wget_output_path
FAVICON_CONFIG = abx.pm.hook.get_CONFIGS().favicon
# TODO: banish this awful duplication from the codebase and import these
# from their respective extractor files
canonical = {

29
archivebox/machine/models.py
View File

@@ -8,9 +8,10 @@ from django.db import models
from django.utils import timezone
from django.utils.functional import cached_property
import abx.archivebox.reads
import abx
import archivebox
from abx.archivebox.base_binary import BaseBinary, BaseBinProvider
from pydantic_pkgr import Binary, BinProvider
from archivebox.abid_utils.models import ABIDModel, ABIDField, AutoDateTimeField, ModelWithHealthStats
from .detect import get_host_guid, get_os_info, get_vm_info, get_host_network, get_host_stats
@@ -180,7 +181,7 @@ class NetworkInterface(ABIDModel, ModelWithHealthStats):
class InstalledBinaryManager(models.Manager):
def get_from_db_or_cache(self, binary: BaseBinary) -> 'InstalledBinary':
def get_from_db_or_cache(self, binary: Binary) -> 'InstalledBinary':
"""Get or create an InstalledBinary record for a Binary on the local machine"""
global _CURRENT_BINARIES
@@ -216,7 +217,7 @@ class InstalledBinaryManager(models.Manager):
# if binary was not yet loaded from filesystem, do it now
# this is expensive, we have to find it's abspath, version, and sha256, but it's necessary
# to make sure we have a good, up-to-date record of it in the DB & in-memroy cache
binary = binary.load(fresh=True)
binary = archivebox.pm.hook.binary_load(binary=binary, fresh=True)
assert binary.loaded_binprovider and binary.loaded_abspath and binary.loaded_version and binary.loaded_sha256, f'Failed to load binary {binary.name} abspath, version, and sha256'
@@ -291,8 +292,8 @@ class InstalledBinary(ABIDModel, ModelWithHealthStats):
if not hasattr(self, 'machine'):
self.machine = Machine.objects.current()
if not self.binprovider:
all_known_binproviders = list(abx.archivebox.reads.get_BINPROVIDERS().values())
binary = BaseBinary(name=self.name, binproviders=all_known_binproviders).load(fresh=True)
all_known_binproviders = list(abx.as_dict(archivebox.pm.hook.get_BINPROVIDERS()).values())
binary = archivebox.pm.hook.binary_load(binary=Binary(name=self.name, binproviders=all_known_binproviders), fresh=True)
self.binprovider = binary.loaded_binprovider.name if binary.loaded_binprovider else None
if not self.abspath:
self.abspath = self.BINPROVIDER.get_abspath(self.name)
@@ -304,16 +305,16 @@ class InstalledBinary(ABIDModel, ModelWithHealthStats):
super().clean(*args, **kwargs)
@cached_property
def BINARY(self) -> BaseBinary:
for binary in abx.archivebox.reads.get_BINARIES().values():
def BINARY(self) -> Binary:
for binary in abx.as_dict(archivebox.pm.hook.get_BINARIES()).values():
if binary.name == self.name:
return binary
raise Exception(f'Orphaned InstalledBinary {self.name} {self.binprovider} was found in DB, could not find any plugin that defines it')
# TODO: we could technically reconstruct it from scratch, but why would we ever want to do that?
@cached_property
def BINPROVIDER(self) -> BaseBinProvider:
for binprovider in abx.archivebox.reads.get_BINPROVIDERS().values():
def BINPROVIDER(self) -> BinProvider:
for binprovider in abx.as_dict(archivebox.pm.hook.get_BINPROVIDERS()).values():
if binprovider.name == self.binprovider:
return binprovider
raise Exception(f'Orphaned InstalledBinary(name={self.name}) was found in DB, could not find any plugin that defines BinProvider(name={self.binprovider})')
@@ -321,7 +322,7 @@ class InstalledBinary(ABIDModel, ModelWithHealthStats):
# maybe not a good idea to provide this? Binary in DB is a record of the binary's config
# whereas a loaded binary is a not-yet saved instance that may not have the same config
# why would we want to load a binary record from the db when it could be freshly loaded?
def load_from_db(self) -> BaseBinary:
def load_from_db(self) -> Binary:
# TODO: implement defaults arg in pydantic_pkgr
# return self.BINARY.load(defaults={
# 'binprovider': self.BINPROVIDER,
@@ -330,7 +331,7 @@ class InstalledBinary(ABIDModel, ModelWithHealthStats):
# 'sha256': self.sha256,
# })
return BaseBinary.model_validate({
return Binary.model_validate({
**self.BINARY.model_dump(),
'abspath': self.abspath and Path(self.abspath),
'version': self.version,
@@ -340,5 +341,5 @@ class InstalledBinary(ABIDModel, ModelWithHealthStats):
'overrides': self.BINARY.overrides,
})
def load_fresh(self) -> BaseBinary:
return self.BINARY.load(fresh=True)
def load_fresh(self) -> Binary:
return archivebox.pm.hook.binary_load(binary=self.BINARY, fresh=True)

60
archivebox/main.py
View File

@@ -14,6 +14,10 @@ from crontab import CronTab, CronSlices
from django.db.models import QuerySet
from django.utils import timezone
from pydantic_pkgr import Binary
import abx
import archivebox
from archivebox.misc.checks import check_data_folder
from archivebox.misc.util import enforce_types # type: ignore
from archivebox.misc.system import get_dir_size, dedupe_cron_jobs, CRON_COMMENT
@@ -197,13 +201,13 @@ def version(quiet: bool=False,
from django.conf import settings
from abx.archivebox.base_binary import BaseBinary, apt, brew, env
from abx_plugin_default_binproviders import apt, brew, env
from archivebox.config.version import get_COMMIT_HASH, get_BUILD_TIME
from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP, RUNNING_AS_UID, RUNNING_AS_GID
from archivebox.config.paths import get_data_locations, get_code_locations
from plugins_auth.ldap.config import LDAP_CONFIG
LDAP_ENABLED = archivebox.pm.hook.get_FLAT_CONFIG().LDAP_ENABLED
# 0.7.1
@@ -242,7 +246,7 @@ def version(quiet: bool=False,
f'SUDO={CONSTANTS.IS_ROOT}',
f'ID={CONSTANTS.MACHINE_ID}:{CONSTANTS.COLLECTION_ID}',
f'SEARCH_BACKEND={SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE}',
f'LDAP={LDAP_CONFIG.LDAP_ENABLED}',
f'LDAP={LDAP_ENABLED}',
#f'DB=django.db.backends.sqlite3 (({CONFIG["SQLITE_JOURNAL_MODE"]})', # add this if we have more useful info to show eventually
)
prnt()
@@ -264,7 +268,8 @@ def version(quiet: bool=False,
prnt('[pale_green1][i] Binary Dependencies:[/pale_green1]')
failures = []
for name, binary in list(settings.BINARIES.items()):
BINARIES = abx.as_dict(archivebox.pm.hook.get_BINARIES())
for name, binary in list(BINARIES.items()):
if binary.name == 'archivebox':
continue
@@ -295,14 +300,15 @@ def version(quiet: bool=False,
prnt()
prnt('[gold3][i] Package Managers:[/gold3]')
for name, binprovider in list(settings.BINPROVIDERS.items()):
BINPROVIDERS = abx.as_dict(archivebox.pm.hook.get_BINPROVIDERS())
for name, binprovider in list(BINPROVIDERS.items()):
err = None
if binproviders and binprovider.name not in binproviders:
continue
# TODO: implement a BinProvider.BINARY() method that gets the loaded binary for a binprovider's INSTALLER_BIN
loaded_bin = binprovider.INSTALLER_BINARY or BaseBinary(name=binprovider.INSTALLER_BIN, binproviders=[env, apt, brew])
loaded_bin = binprovider.INSTALLER_BINARY or Binary(name=binprovider.INSTALLER_BIN, binproviders=[env, apt, brew])
abspath = None
if loaded_bin.abspath:
@@ -1050,10 +1056,7 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
# - recommend user re-run with sudo if any deps need to be installed as root
from rich import print
from django.conf import settings
import abx.archivebox.reads
from archivebox.config.permissions import IS_ROOT, ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
from archivebox.config.paths import get_or_create_working_lib_dir
@@ -1076,11 +1079,11 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
package_manager_names = ', '.join(
f'[yellow]{binprovider.name}[/yellow]'
for binprovider in reversed(list(abx.archivebox.reads.get_BINPROVIDERS().values()))
for binprovider in reversed(list(abx.as_dict(abx.pm.hook.get_BINPROVIDERS()).values()))
if not binproviders or (binproviders and binprovider.name in binproviders)
)
print(f'[+] Setting up package managers {package_manager_names}...')
for binprovider in reversed(list(abx.archivebox.reads.get_BINPROVIDERS().values())):
for binprovider in reversed(list(abx.as_dict(abx.pm.hook.get_BINPROVIDERS()).values())):
if binproviders and binprovider.name not in binproviders:
continue
try:
@@ -1093,7 +1096,7 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
print()
for binary in reversed(list(abx.archivebox.reads.get_BINARIES().values())):
for binary in reversed(list(abx.as_dict(abx.pm.hook.get_BINARIES()).values())):
if binary.name in ('archivebox', 'django', 'sqlite', 'python'):
# obviously must already be installed if we are running
continue
@@ -1123,7 +1126,8 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
result = binary.install(binproviders=[binprovider_name], dry_run=dry_run).model_dump(exclude={'overrides', 'bin_dir', 'hook_type'})
sys.stderr.write("\033[00m\n") # reset
else:
result = binary.load_or_install(binproviders=[binprovider_name], fresh=True, dry_run=dry_run, quiet=False).model_dump(exclude={'overrides', 'bin_dir', 'hook_type'})
loaded_binary = archivebox.pm.hook.binary_load_or_install(binary=binary, binproviders=[binprovider_name], fresh=True, dry_run=dry_run, quiet=False)
result = loaded_binary.model_dump(exclude={'overrides', 'bin_dir', 'hook_type'})
if result and result['loaded_version']:
break
except Exception as e:
@@ -1134,7 +1138,8 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
binary.install(dry_run=dry_run).model_dump(exclude={'overrides', 'bin_dir', 'hook_type'})
sys.stderr.write("\033[00m\n") # reset
else:
binary.load_or_install(fresh=True, dry_run=dry_run).model_dump(exclude={'overrides', 'bin_dir', 'hook_type'})
loaded_binary = archivebox.pm.hook.binary_load_or_install(binary=binary, fresh=True, dry_run=dry_run)
result = loaded_binary.model_dump(exclude={'overrides', 'bin_dir', 'hook_type'})
if IS_ROOT and LIB_DIR:
with SudoPermission(uid=0):
if ARCHIVEBOX_USER == 0:
@@ -1158,7 +1163,7 @@ def install(out_dir: Path=DATA_DIR, binproviders: Optional[List[str]]=None, bina
print('\n[green][√] Set up ArchiveBox and its dependencies successfully.[/green]\n', file=sys.stderr)
from plugins_pkg.pip.binaries import ARCHIVEBOX_BINARY
from abx_plugin_pip.binaries import ARCHIVEBOX_BINARY
extra_args = []
if binproviders:
@@ -1184,8 +1189,6 @@ def config(config_options_str: Optional[str]=None,
out_dir: Path=DATA_DIR) -> None:
"""Get and set your ArchiveBox project configuration values"""
import abx.archivebox.reads
from rich import print
check_data_folder()
@@ -1199,7 +1202,8 @@ def config(config_options_str: Optional[str]=None,
elif config_options_str:
config_options = config_options_str.split('\n')
from django.conf import settings
FLAT_CONFIG = archivebox.pm.hook.get_FLAT_CONFIG()
CONFIGS = archivebox.pm.hook.get_CONFIGS()
config_options = config_options or []
@@ -1209,8 +1213,8 @@ def config(config_options_str: Optional[str]=None,
if search:
if config_options:
config_options = [get_real_name(key) for key in config_options]
matching_config = {key: settings.FLAT_CONFIG[key] for key in config_options if key in settings.FLAT_CONFIG}
for config_section in settings.CONFIGS.values():
matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
for config_section in CONFIGS.values():
aliases = config_section.aliases
for search_key in config_options:
@@ -1229,15 +1233,15 @@ def config(config_options_str: Optional[str]=None,
elif get or no_args:
if config_options:
config_options = [get_real_name(key) for key in config_options]
matching_config = {key: settings.FLAT_CONFIG[key] for key in config_options if key in settings.FLAT_CONFIG}
failed_config = [key for key in config_options if key not in settings.FLAT_CONFIG]
matching_config = {key: FLAT_CONFIG[key] for key in config_options if key in FLAT_CONFIG}
failed_config = [key for key in config_options if key not in FLAT_CONFIG]
if failed_config:
stderr()
stderr('[X] These options failed to get', color='red')
stderr(' {}'.format('\n '.join(config_options)))
raise SystemExit(1)
else:
matching_config = settings.FLAT_CONFIG
matching_config = FLAT_CONFIG
print(printable_config(matching_config))
raise SystemExit(not matching_config)
@@ -1258,20 +1262,20 @@ def config(config_options_str: Optional[str]=None,
if key != raw_key:
stderr(f'[i] Note: The config option {raw_key} has been renamed to {key}, please use the new name going forwards.', color='lightyellow')
if key in settings.FLAT_CONFIG:
if key in FLAT_CONFIG:
new_config[key] = val.strip()
else:
failed_options.append(line)
if new_config:
before = settings.FLAT_CONFIG
before = FLAT_CONFIG
matching_config = write_config_file(new_config)
after = {**load_all_config(), **abx.archivebox.reads.get_FLAT_CONFIG()}
after = {**load_all_config(), **archivebox.pm.hook.get_FLAT_CONFIG()}
print(printable_config(matching_config))
side_effect_changes = {}
for key, val in after.items():
if key in settings.FLAT_CONFIG and (str(before[key]) != str(after[key])) and (key not in matching_config):
if key in FLAT_CONFIG and (str(before[key]) != str(after[key])) and (key not in matching_config):
side_effect_changes[key] = after[key]
# import ipdb; ipdb.set_trace()
@@ -1313,7 +1317,7 @@ def schedule(add: bool=False,
"""Set ArchiveBox to regularly import URLs at specific times using cron"""
check_data_folder()
from archivebox.plugins_pkg.pip.binaries import ARCHIVEBOX_BINARY
from abx_plugin_pip.binaries import ARCHIVEBOX_BINARY
from archivebox.config.permissions import USER
Path(CONSTANTS.LOGS_DIR).mkdir(exist_ok=True)

3
archivebox/misc/checks.py
View File

@@ -201,6 +201,7 @@ def check_tmp_dir(tmp_dir=None, throw=False, quiet=False, must_exist=True):
def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_exist=True):
import archivebox
from archivebox.config.permissions import ARCHIVEBOX_USER, ARCHIVEBOX_GROUP
from archivebox.misc.logging import STDERR
from archivebox.config.paths import dir_is_writable, get_or_create_working_lib_dir
@@ -209,6 +210,8 @@ def check_lib_dir(lib_dir: Path | None = None, throw=False, quiet=False, must_ex
lib_dir = lib_dir or STORAGE_CONFIG.LIB_DIR
assert lib_dir == archivebox.pm.hook.get_LIB_DIR(), "lib_dir is not the same as the one in the flat config"
if not must_exist and not os.path.isdir(lib_dir):
return True

3
archivebox/misc/shell_welcome_message.py
View File

@@ -23,7 +23,7 @@ from archivebox import CONSTANTS # noqa
from ..main import * # noqa
from ..cli import CLI_SUBCOMMANDS
CONFIG = settings.FLAT_CONFIG
CONFIG = archivebox.pm.hook.get_FLAT_CONFIG()
CLI_COMMAND_NAMES = ", ".join(CLI_SUBCOMMANDS.keys())
if __name__ == '__main__':
@@ -55,6 +55,5 @@ if __name__ == '__main__':
prnt(' add[blink][deep_sky_blue4]?[/deep_sky_blue4][/blink] [grey53]# add ? after anything to get help[/]')
prnt(' add("https://example.com/some/new/url") [grey53]# call CLI methods from the shell[/]')
prnt(' snap = Snapshot.objects.filter(url__contains="https://example.com").last() [grey53]# query for individual snapshots[/]')
prnt(' archivebox.plugins_extractor.wget.apps.WGET_EXTRACTOR.extract(snap.id) [grey53]# call an extractor directly[/]')
prnt(' snap.archiveresult_set.all() [grey53]# see extractor results[/]')
prnt(' bool(re.compile(CONFIG.URL_DENYLIST).search("https://example.com/abc.exe")) [grey53]# test out a config change[/]')

11
archivebox/parsers/pocket_api.py
View File

@@ -6,8 +6,7 @@ import re
from typing import IO, Iterable, Optional
from configparser import ConfigParser
from pocket import Pocket
import archivebox
from archivebox.config import CONSTANTS
from archivebox.misc.util import enforce_types
from archivebox.misc.system import atomic_write
@@ -22,7 +21,7 @@ API_DB_PATH = CONSTANTS.SOURCES_DIR / 'pocket_api.db'
_BROKEN_PROTOCOL_RE = re.compile('^(http[s]?)(:/(?!/))')
def get_pocket_articles(api: Pocket, since=None, page=0):
def get_pocket_articles(api, since=None, page=0):
body, headers = api.get(
state='archive',
sort='oldest',
@@ -94,7 +93,9 @@ def should_parse_as_pocket_api(text: str) -> bool:
def parse_pocket_api_export(input_buffer: IO[str], **_kwargs) -> Iterable[Link]:
"""Parse bookmarks from the Pocket API"""
from archivebox.plugins_extractor.pocket.config import POCKET_CONFIG
from pocket import Pocket
FLAT_CONFIG = archivebox.pm.hook.get_FLAT_CONFIG()
input_buffer.seek(0)
pattern = re.compile(r"^pocket:\/\/(\w+)")
@@ -102,7 +103,7 @@ def parse_pocket_api_export(input_buffer: IO[str], **_kwargs) -> Iterable[Link]:
if should_parse_as_pocket_api(line):
username = pattern.search(line).group(1)
api = Pocket(POCKET_CONFIG.POCKET_CONSUMER_KEY, POCKET_CONFIG.POCKET_ACCESS_TOKENS[username])
api = Pocket(FLAT_CONFIG.POCKET_CONSUMER_KEY, FLAT_CONFIG.POCKET_ACCESS_TOKENS[username])
api.last_since = None
for article in get_pocket_articles(api, since=read_since(username)):

25
archivebox/parsers/readwise_reader_api.py
View File

@@ -8,9 +8,10 @@ from datetime import datetime
from typing import IO, Iterable, Optional
from configparser import ConfigParser
import abx
from archivebox.misc.util import enforce_types
from archivebox.misc.system import atomic_write
from archivebox.plugins_extractor.readwise.config import READWISE_CONFIG
from ..index.schema import Link
@@ -62,26 +63,30 @@ def link_from_article(article: dict, sources: list):
def write_cursor(username: str, since: str):
if not READWISE_CONFIG.READWISE_DB_PATH.exists():
atomic_write(READWISE_CONFIG.READWISE_DB_PATH, "")
READWISE_DB_PATH = abx.pm.hook.get_CONFIG().READWISE_DB_PATH
if not READWISE_DB_PATH.exists():
atomic_write(READWISE_DB_PATH, "")
since_file = ConfigParser()
since_file.optionxform = str
since_file.read(READWISE_CONFIG.READWISE_DB_PATH)
since_file.read(READWISE_DB_PATH)
since_file[username] = {"since": since}
with open(READWISE_CONFIG.READWISE_DB_PATH, "w+") as new:
with open(READWISE_DB_PATH, "w+") as new:
since_file.write(new)
def read_cursor(username: str) -> Optional[str]:
if not READWISE_CONFIG.READWISE_DB_PATH.exists():
atomic_write(READWISE_CONFIG.READWISE_DB_PATH, "")
READWISE_DB_PATH = abx.pm.hook.get_CONFIG().READWISE_DB_PATH
if not READWISE_DB_PATH.exists():
atomic_write(READWISE_DB_PATH, "")
config_file = ConfigParser()
config_file.optionxform = str
config_file.read(READWISE_CONFIG.READWISE_DB_PATH)
config_file.read(READWISE_DB_PATH)
return config_file.get(username, "since", fallback=None)
@@ -97,12 +102,14 @@ def should_parse_as_readwise_reader_api(text: str) -> bool:
def parse_readwise_reader_api_export(input_buffer: IO[str], **_kwargs) -> Iterable[Link]:
"""Parse bookmarks from the Readwise Reader API"""
READWISE_READER_TOKENS = abx.pm.hook.get_CONFIG().READWISE_READER_TOKENS
input_buffer.seek(0)
pattern = re.compile(r"^readwise-reader:\/\/(\w+)")
for line in input_buffer:
if should_parse_as_readwise_reader_api(line):
username = pattern.search(line).group(1)
api = ReadwiseReaderAPI(READWISE_CONFIG.READWISE_READER_TOKENS[username], cursor=read_cursor(username))
api = ReadwiseReaderAPI(READWISE_READER_TOKENS[username], cursor=read_cursor(username))
for article in get_readwise_reader_articles(api):
yield link_from_article(article, sources=[line])

6
archivebox/search/__init__.py
View File

@@ -6,8 +6,8 @@ from typing import List, Union
from django.db.models import QuerySet
from django.conf import settings
import abx.archivebox.reads
import abx
import archivebox
from archivebox.index.schema import Link
from archivebox.misc.util import enforce_types
from archivebox.misc.logging import stderr
@@ -57,7 +57,7 @@ def get_indexable_content(results: QuerySet):
def import_backend():
for backend in abx.archivebox.reads.get_SEARCHBACKENDS().values():
for backend in abx.as_dict(archivebox.pm.hook.get_SEARCHBACKENDS()).values():
if backend.name == SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE:
return backend
raise Exception(f'Could not load {SEARCH_BACKEND_CONFIG.SEARCH_BACKEND_ENGINE} as search backend')

28
archivebox/vendor/__init__.py vendored
View File

@@ -4,23 +4,27 @@ from pathlib import Path
VENDOR_DIR = Path(__file__).parent
VENDORED_LIBS = {
# sys.path dir: library name
#'python-atomicwrites': 'atomicwrites',
#'django-taggit': 'taggit',
# 'pydantic-pkgr': 'pydantic_pkgr',
# 'pocket': 'pocket',
#'base32-crockford': 'base32_crockford',
}
VENDORED_LIBS = [
'abx',
'pydantic-pkgr',
'pocket',
]
for subdir in reversed(sorted(VENDOR_DIR.iterdir())):
if subdir.is_dir() and subdir.name not in VENDORED_LIBS and not subdir.name.startswith('_'):
VENDORED_LIBS.append(subdir.name)
def load_vendored_libs():
for lib_subdir, lib_name in VENDORED_LIBS.items():
lib_dir = VENDOR_DIR / lib_subdir
assert lib_dir.is_dir(), 'Expected vendor libary {lib_name} could not be found in {lib_dir}'
if str(VENDOR_DIR) not in sys.path:
sys.path.append(str(VENDOR_DIR))
for lib_name in VENDORED_LIBS:
lib_dir = VENDOR_DIR / lib_name
assert lib_dir.is_dir(), f'Expected vendor libary {lib_name} could not be found in {lib_dir}'
try:
lib = importlib.import_module(lib_name)
# print(f"Successfully imported lib from environment {lib_name}: {inspect.getfile(lib)}")
# print(f"Successfully imported lib from environment {lib_name}")
except ImportError:
sys.path.append(str(lib_dir))
try:

0
archivebox/vendor/abx-plugin-archivedotorg/README.md vendored Normal file
View File

21
archivebox/vendor/abx-plugin-archivedotorg/abx_plugin_archivedotorg/__init__.py vendored Normal file
View File

@@ -0,0 +1,21 @@
__label__ = 'Archive.org'
__homepage__ = 'https://archive.org'
import abx
@abx.hookimpl
def get_CONFIG():
from .config import ARCHIVEDOTORG_CONFIG
return {
'ARCHIVEDOTORG_CONFIG': ARCHIVEDOTORG_CONFIG
}
# @abx.hookimpl
# def get_EXTRACTORS():
# from .extractors import ARCHIVEDOTORG_EXTRACTOR
#
# return {
# 'archivedotorg': ARCHIVEDOTORG_EXTRACTOR,
# }

0
archivebox/extractors/archive_org.py → archivebox/vendor/abx-plugin-archivedotorg/abx_plugin_archivedotorg/archive_org.py vendored
View File

8
archivebox/vendor/abx-plugin-archivedotorg/abx_plugin_archivedotorg/config.py vendored Normal file
View File

@@ -0,0 +1,8 @@
from abx_spec_config.base_configset import BaseConfigSet
class ArchivedotorgConfig(BaseConfigSet):
SAVE_ARCHIVE_DOT_ORG: bool = True
ARCHIVEDOTORG_CONFIG = ArchivedotorgConfig()

18
archivebox/vendor/abx-plugin-archivedotorg/pyproject.toml vendored Normal file
View File

@@ -0,0 +1,18 @@
[project]
name = "abx-plugin-archivedotorg"
version = "2024.10.28"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"abx-spec-config>=0.1.0",
"abx-plugin-curl>=2024.10.24",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_archivedotorg = "abx_plugin_archivedotorg"

0
archivebox/vendor/abx-plugin-chrome/README.md vendored Normal file
View File

34
archivebox/vendor/abx-plugin-chrome/abx_plugin_chrome/__init__.py vendored Normal file
View File

@@ -0,0 +1,34 @@
__label__ = 'Chrome'
__author__ = 'ArchiveBox'
import abx
@abx.hookimpl
def get_CONFIG():
from .config import CHROME_CONFIG
return {
'CHROME_CONFIG': CHROME_CONFIG
}
@abx.hookimpl
def get_BINARIES():
from .binaries import CHROME_BINARY
return {
'chrome': CHROME_BINARY,
}
@abx.hookimpl
def ready():
from .config import CHROME_CONFIG
CHROME_CONFIG.validate()
# @abx.hookimpl
# def get_EXTRACTORS():
# return {
# 'pdf': PDF_EXTRACTOR,
# 'screenshot': SCREENSHOT_EXTRACTOR,
# 'dom': DOM_EXTRACTOR,
# }

149
archivebox/vendor/abx-plugin-chrome/abx_plugin_chrome/binaries.py vendored Normal file
View File

@@ -0,0 +1,149 @@
import os
import platform
from pathlib import Path
from typing import List, Optional
from pydantic import InstanceOf
from pydantic_pkgr import (
Binary,
BinProvider,
BinName,
BinaryOverrides,
bin_abspath,
)
import abx
from abx_plugin_default_binproviders import apt, brew, env
from abx_plugin_puppeteer.binproviders import PUPPETEER_BINPROVIDER
from abx_plugin_playwright.binproviders import PLAYWRIGHT_BINPROVIDER
from .config import CHROME_CONFIG
CHROMIUM_BINARY_NAMES_LINUX = [
"chromium",
"chromium-browser",
"chromium-browser-beta",
"chromium-browser-unstable",
"chromium-browser-canary",
"chromium-browser-dev",
]
CHROMIUM_BINARY_NAMES_MACOS = ["/Applications/Chromium.app/Contents/MacOS/Chromium"]
CHROMIUM_BINARY_NAMES = CHROMIUM_BINARY_NAMES_LINUX + CHROMIUM_BINARY_NAMES_MACOS
CHROME_BINARY_NAMES_LINUX = [
"google-chrome",
"google-chrome-stable",
"google-chrome-beta",
"google-chrome-canary",
"google-chrome-unstable",
"google-chrome-dev",
"chrome"
]
CHROME_BINARY_NAMES_MACOS = [
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
]
CHROME_BINARY_NAMES = CHROME_BINARY_NAMES_LINUX + CHROME_BINARY_NAMES_MACOS
CHROME_APT_DEPENDENCIES = [
'apt-transport-https', 'at-spi2-common',
'fontconfig', 'fonts-freefont-ttf', 'fonts-ipafont-gothic', 'fonts-kacst', 'fonts-khmeros', 'fonts-liberation', 'fonts-noto', 'fonts-noto-color-emoji', 'fonts-symbola', 'fonts-thai-tlwg', 'fonts-tlwg-loma-otf', 'fonts-unifont', 'fonts-wqy-zenhei',
'libasound2', 'libatk-bridge2.0-0', 'libatk1.0-0', 'libatspi2.0-0', 'libavahi-client3', 'libavahi-common-data', 'libavahi-common3', 'libcairo2', 'libcups2',
'libdbus-1-3', 'libdrm2', 'libfontenc1', 'libgbm1', 'libglib2.0-0', 'libice6', 'libnspr4', 'libnss3', 'libsm6', 'libunwind8', 'libx11-6', 'libxaw7', 'libxcb1',
'libxcomposite1', 'libxdamage1', 'libxext6', 'libxfixes3', 'libxfont2', 'libxkbcommon0', 'libxkbfile1', 'libxmu6', 'libxpm4', 'libxrandr2', 'libxt6', 'x11-utils', 'x11-xkb-utils', 'xfonts-encodings',
'chromium-browser',
]
def autodetect_system_chrome_install(PATH=None) -> Optional[Path]:
for bin_name in CHROME_BINARY_NAMES + CHROMIUM_BINARY_NAMES:
abspath = bin_abspath(bin_name, PATH=env.PATH)
if abspath:
return abspath
return None
def create_macos_app_symlink(target: Path, shortcut: Path):
"""
on macOS, some binaries are inside of .app, so we need to
create a tiny bash script instead of a symlink
(so that ../ parent relationships are relative to original .app instead of callsite dir)
"""
# TODO: should we enforce this? is it useful in any other situation?
# if platform.system().lower() != 'darwin':
# raise Exception(...)
shortcut.unlink(missing_ok=True)
shortcut.write_text(f"""#!/usr/bin/env bash\nexec '{target}' "$@"\n""")
shortcut.chmod(0o777) # make sure its executable by everyone
###################### Config ##########################
class ChromeBinary(Binary):
name: BinName = CHROME_CONFIG.CHROME_BINARY
binproviders_supported: List[InstanceOf[BinProvider]] = [PUPPETEER_BINPROVIDER, env, PLAYWRIGHT_BINPROVIDER, apt, brew]
overrides: BinaryOverrides = {
env.name: {
'abspath': lambda: autodetect_system_chrome_install(PATH=env.PATH), # /usr/bin/google-chrome-stable
},
PUPPETEER_BINPROVIDER.name: {
'packages': ['chrome@stable'], # npx @puppeteer/browsers install chrome@stable
},
PLAYWRIGHT_BINPROVIDER.name: {
'packages': ['chromium'], # playwright install chromium
},
apt.name: {
'packages': CHROME_APT_DEPENDENCIES,
},
brew.name: {
'packages': ['--cask', 'chromium'] if platform.system().lower() == 'darwin' else [],
},
}
@staticmethod
def symlink_to_lib(binary, bin_dir=None) -> None:
bin_dir = bin_dir or abx.pm.hook.get_BIN_DIR()
if not (binary.abspath and os.path.isfile(binary.abspath)):
return
bin_dir.mkdir(parents=True, exist_ok=True)
symlink = bin_dir / binary.name
try:
if platform.system().lower() == 'darwin':
# if on macOS, browser binary is inside a .app, so we need to create a tiny bash script instead of a symlink
create_macos_app_symlink(binary.abspath, symlink)
else:
# otherwise on linux we can symlink directly to binary executable
symlink.unlink(missing_ok=True)
symlink.symlink_to(binary.abspath)
except Exception:
# print(f'[red]:warning: Failed to symlink {symlink} -> {binary.abspath}[/red] {err}')
# not actually needed, we can just run without it
pass
@staticmethod
def chrome_cleanup_lockfile():
"""
Cleans up any state or runtime files that chrome leaves behind when killed by
a timeout or other error
"""
try:
linux_lock_file = Path("~/.config/chromium/SingletonLock").expanduser()
linux_lock_file.unlink(missing_ok=True)
except Exception:
pass
if CHROME_CONFIG.CHROME_USER_DATA_DIR:
try:
(CHROME_CONFIG.CHROME_USER_DATA_DIR / 'SingletonLock').unlink(missing_ok=True)
except Exception:
pass
CHROME_BINARY = ChromeBinary()

201
archivebox/vendor/abx-plugin-chrome/abx_plugin_chrome/config.py vendored Normal file
View File

@@ -0,0 +1,201 @@
import os
from pathlib import Path
from typing import List, Optional
from pydantic import Field
from pydantic_pkgr import bin_abspath
from abx_spec_config.base_configset import BaseConfigSet
from abx_plugin_default_binproviders import env
from archivebox.config import CONSTANTS
from archivebox.config.common import ARCHIVING_CONFIG, SHELL_CONFIG
from archivebox.misc.logging import STDERR
from archivebox.misc.util import dedupe
from archivebox.logging_util import pretty_path
CHROMIUM_BINARY_NAMES_LINUX = [
"chromium",
"chromium-browser",
"chromium-browser-beta",
"chromium-browser-unstable",
"chromium-browser-canary",
"chromium-browser-dev",
]
CHROMIUM_BINARY_NAMES_MACOS = ["/Applications/Chromium.app/Contents/MacOS/Chromium"]
CHROMIUM_BINARY_NAMES = CHROMIUM_BINARY_NAMES_LINUX + CHROMIUM_BINARY_NAMES_MACOS
CHROME_BINARY_NAMES_LINUX = [
"google-chrome",
"google-chrome-stable",
"google-chrome-beta",
"google-chrome-canary",
"google-chrome-unstable",
"google-chrome-dev",
"chrome"
]
CHROME_BINARY_NAMES_MACOS = [
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
]
CHROME_BINARY_NAMES = CHROME_BINARY_NAMES_LINUX + CHROME_BINARY_NAMES_MACOS
APT_DEPENDENCIES = [
'apt-transport-https', 'at-spi2-common', 'chromium-browser',
'fontconfig', 'fonts-freefont-ttf', 'fonts-ipafont-gothic', 'fonts-kacst', 'fonts-khmeros', 'fonts-liberation', 'fonts-noto', 'fonts-noto-color-emoji', 'fonts-symbola', 'fonts-thai-tlwg', 'fonts-tlwg-loma-otf', 'fonts-unifont', 'fonts-wqy-zenhei',
'libasound2', 'libatk-bridge2.0-0', 'libatk1.0-0', 'libatspi2.0-0', 'libavahi-client3', 'libavahi-common-data', 'libavahi-common3', 'libcairo2', 'libcups2',
'libdbus-1-3', 'libdrm2', 'libfontenc1', 'libgbm1', 'libglib2.0-0', 'libice6', 'libnspr4', 'libnss3', 'libsm6', 'libunwind8', 'libx11-6', 'libxaw7', 'libxcb1',
'libxcomposite1', 'libxdamage1', 'libxext6', 'libxfixes3', 'libxfont2', 'libxkbcommon0', 'libxkbfile1', 'libxmu6', 'libxpm4', 'libxrandr2', 'libxt6', 'x11-utils', 'x11-xkb-utils', 'xfonts-encodings',
]
def autodetect_system_chrome_install(PATH=None) -> Optional[Path]:
for bin_name in CHROME_BINARY_NAMES + CHROMIUM_BINARY_NAMES:
abspath = bin_abspath(bin_name, PATH=env.PATH)
if abspath:
return abspath
return None
def create_macos_app_symlink(target: Path, shortcut: Path):
"""
on macOS, some binaries are inside of .app, so we need to
create a tiny bash script instead of a symlink
(so that ../ parent relationships are relative to original .app instead of callsite dir)
"""
# TODO: should we enforce this? is it useful in any other situation?
# if platform.system().lower() != 'darwin':
# raise Exception(...)
shortcut.unlink(missing_ok=True)
shortcut.write_text(f"""#!/usr/bin/env bash\nexec '{target}' "$@"\n""")
shortcut.chmod(0o777) # make sure its executable by everyone
###################### Config ##########################
class ChromeConfig(BaseConfigSet):
USE_CHROME: bool = Field(default=True)
# Chrome Binary
CHROME_BINARY: str = Field(default='chrome')
CHROME_DEFAULT_ARGS: List[str] = Field(default=[
'--virtual-time-budget=15000',
'--disable-features=DarkMode',
"--run-all-compositor-stages-before-draw",
"--hide-scrollbars",
"--autoplay-policy=no-user-gesture-required",
"--no-first-run",
"--use-fake-ui-for-media-stream",
"--use-fake-device-for-media-stream",
"--simulate-outdated-no-au='Tue, 31 Dec 2099 23:59:59 GMT'",
])
CHROME_EXTRA_ARGS: List[str] = Field(default=[])
# Chrome Options Tuning
CHROME_TIMEOUT: int = Field(default=lambda: ARCHIVING_CONFIG.TIMEOUT - 10)
CHROME_HEADLESS: bool = Field(default=True)
CHROME_SANDBOX: bool = Field(default=lambda: not SHELL_CONFIG.IN_DOCKER)
CHROME_RESOLUTION: str = Field(default=lambda: ARCHIVING_CONFIG.RESOLUTION)
CHROME_CHECK_SSL_VALIDITY: bool = Field(default=lambda: ARCHIVING_CONFIG.CHECK_SSL_VALIDITY)
# Cookies & Auth
CHROME_USER_AGENT: str = Field(default=lambda: ARCHIVING_CONFIG.USER_AGENT)
CHROME_USER_DATA_DIR: Path | None = Field(default=CONSTANTS.PERSONAS_DIR / 'Default' / 'chrome_profile')
CHROME_PROFILE_NAME: str = Field(default='Default')
# Extractor Toggles
SAVE_SCREENSHOT: bool = Field(default=True, alias='FETCH_SCREENSHOT')
SAVE_DOM: bool = Field(default=True, alias='FETCH_DOM')
SAVE_PDF: bool = Field(default=True, alias='FETCH_PDF')
def validate(self):
from archivebox.config.paths import create_and_chown_dir
if self.USE_CHROME and self.CHROME_TIMEOUT < 15:
STDERR.print()
STDERR.print(f'[red][!] Warning: TIMEOUT is set too low! (currently set to TIMEOUT={self.CHROME_TIMEOUT} seconds)[/red]')
STDERR.print(' Chrome will fail to archive all sites if set to less than ~15 seconds.')
STDERR.print(' (Setting it to somewhere between 30 and 300 seconds is recommended)')
STDERR.print()
STDERR.print(' If you want to make ArchiveBox run faster, disable specific archive methods instead:')
STDERR.print(' https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#archive-method-toggles')
STDERR.print()
# if user has specified a user data dir, make sure its valid
if self.USE_CHROME and self.CHROME_USER_DATA_DIR:
try:
create_and_chown_dir(self.CHROME_USER_DATA_DIR / self.CHROME_PROFILE_NAME)
except Exception:
pass
# check to make sure user_data_dir/<profile_name> exists
if not os.path.isdir(self.CHROME_USER_DATA_DIR / self.CHROME_PROFILE_NAME):
STDERR.print()
STDERR.print()
STDERR.print(f'[red][X] Could not find profile "{self.CHROME_PROFILE_NAME}" in CHROME_USER_DATA_DIR.[/red]')
STDERR.print(f' {self.CHROME_USER_DATA_DIR}')
STDERR.print(' Make sure you set it to a Chrome user data directory containing a Default profile folder.')
STDERR.print(' For more info see:')
STDERR.print(' https://github.com/ArchiveBox/ArchiveBox/wiki/Configuration#CHROME_USER_DATA_DIR')
# show special hint if they made the common mistake of putting /Default at the end of the path
if str(self.CHROME_USER_DATA_DIR).replace(str(CONSTANTS.PERSONAS_DIR / 'Default'), '').endswith('/Default'):
STDERR.print()
STDERR.print(' Try removing /Default from the end e.g.:')
STDERR.print(' CHROME_USER_DATA_DIR="{}"'.format(str(self.CHROME_USER_DATA_DIR).rsplit('/Default', 1)[0]))
self.update_in_place(CHROME_USER_DATA_DIR=None)
def chrome_args(self, **options) -> List[str]:
"""helper to build up a chrome shell command with arguments"""
# Chrome CLI flag documentation: https://peter.sh/experiments/chromium-command-line-switches/
options = self.model_copy(update=options)
cmd_args = [*options.CHROME_DEFAULT_ARGS, *options.CHROME_EXTRA_ARGS]
if options.CHROME_HEADLESS:
cmd_args += ["--headless=new"] # expects chrome version >= 111
if not options.CHROME_SANDBOX:
# assume this means we are running inside a docker container
# in docker, GPU support is limited, sandboxing is unecessary,
# and SHM is limited to 64MB by default (which is too low to be usable).
cmd_args += (
"--no-sandbox",
"--no-zygote",
"--disable-dev-shm-usage",
"--disable-software-rasterizer",
"--disable-sync",
# "--password-store=basic",
)
# set window size for screenshot/pdf/etc. rendering
cmd_args += ('--window-size={}'.format(options.CHROME_RESOLUTION),)
if not options.CHROME_CHECK_SSL_VALIDITY:
cmd_args += ('--disable-web-security', '--ignore-certificate-errors')
if options.CHROME_USER_AGENT:
cmd_args += ('--user-agent={}'.format(options.CHROME_USER_AGENT),)
# this no longer works on newer chrome version for some reason, just causes chrome to hang indefinitely:
# if options.CHROME_TIMEOUT:
# cmd_args += ('--timeout={}'.format(options.CHROME_TIMEOUT * 1000),)
if options.CHROME_USER_DATA_DIR:
cmd_args.append('--user-data-dir={}'.format(options.CHROME_USER_DATA_DIR))
cmd_args.append('--profile-directory={}'.format(options.CHROME_PROFILE_NAME or 'Default'))
if not os.path.isfile(options.CHROME_USER_DATA_DIR / options.CHROME_PROFILE_NAME / 'Preferences'):
STDERR.print(f'[green] + creating new Chrome profile in: {pretty_path(options.CHROME_USER_DATA_DIR / options.CHROME_PROFILE_NAME)}[/green]')
cmd_args.remove('--no-first-run')
cmd_args.append('--first-run')
return dedupe(cmd_args)
CHROME_CONFIG = ChromeConfig()

0
archivebox/extractors/dom.py → archivebox/vendor/abx-plugin-chrome/abx_plugin_chrome/dom.py vendored
View File

0
archivebox/extractors/pdf.py → archivebox/vendor/abx-plugin-chrome/abx_plugin_chrome/pdf.py vendored
View File

0
archivebox/extractors/screenshot.py → archivebox/vendor/abx-plugin-chrome/abx_plugin_chrome/screenshot.py vendored
View File

18
archivebox/vendor/abx-plugin-chrome/pyproject.toml vendored Normal file
View File

@@ -0,0 +1,18 @@
[project]
name = "abx-plugin-chrome"
version = "2024.10.28"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"abx-spec-config>=0.1.0",
"abx-spec-pydantic-pkgr>=0.1.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_chrome = "abx_plugin_chrome"

0
archivebox/vendor/abx-plugin-curl/README.md vendored Normal file
View File

18
archivebox/vendor/abx-plugin-curl/abx_plugin_curl/__init__.py vendored Normal file
View File

@@ -0,0 +1,18 @@
import abx
@abx.hookimpl
def get_CONFIG():
from .config import CURL_CONFIG
return {
'curl': CURL_CONFIG
}
@abx.hookimpl
def get_BINARIES():
from .binaries import CURL_BINARY
return {
'curl': CURL_BINARY,
}

18
archivebox/vendor/abx-plugin-curl/abx_plugin_curl/binaries.py vendored Normal file
View File

@@ -0,0 +1,18 @@
__package__ = 'abx_plugin_curl'
from typing import List
from pydantic import InstanceOf
from pydantic_pkgr import BinProvider, BinName, Binary
from abx_plugin_default_binproviders import apt, brew, env
from .config import CURL_CONFIG
class CurlBinary(Binary):
name: BinName = CURL_CONFIG.CURL_BINARY
binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
CURL_BINARY = CurlBinary()

33
archivebox/vendor/abx-plugin-curl/abx_plugin_curl/config.py vendored Normal file
View File

@@ -0,0 +1,33 @@
__package__ = 'abx_plugin_curl'
from typing import List, Optional
from pathlib import Path
from pydantic import Field
from abx_spec_config.base_configset import BaseConfigSet
from archivebox.config.common import ARCHIVING_CONFIG
class CurlConfig(BaseConfigSet):
SAVE_TITLE: bool = Field(default=True)
SAVE_HEADERS: bool = Field(default=True)
USE_CURL: bool = Field(default=True)
CURL_BINARY: str = Field(default='curl')
CURL_ARGS: List[str] = [
'--silent',
'--location',
'--compressed',
]
CURL_EXTRA_ARGS: List[str] = []
CURL_TIMEOUT: int = Field(default=lambda: ARCHIVING_CONFIG.TIMEOUT)
CURL_CHECK_SSL_VALIDITY: bool = Field(default=lambda: ARCHIVING_CONFIG.CHECK_SSL_VALIDITY)
CURL_USER_AGENT: str = Field(default=lambda: ARCHIVING_CONFIG.USER_AGENT)
CURL_COOKIES_FILE: Optional[Path] = Field(default=lambda: ARCHIVING_CONFIG.COOKIES_FILE)
CURL_CONFIG = CurlConfig()

0
archivebox/extractors/headers.py → archivebox/vendor/abx-plugin-curl/abx_plugin_curl/headers.py vendored
View File

18
archivebox/vendor/abx-plugin-curl/pyproject.toml vendored Normal file
View File

@@ -0,0 +1,18 @@
[project]
name = "abx-plugin-curl"
version = "2024.10.24"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"abx-spec-config>=0.1.0",
"abx-spec-pydantic-pkgr>=0.1.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_curl = "abx_plugin_curl"

0
archivebox/vendor/abx-plugin-default-binproviders/README.md vendored Normal file
View File

23
archivebox/vendor/abx-plugin-default-binproviders/abx_plugin_default_binproviders.py vendored Normal file
View File

@@ -0,0 +1,23 @@
import abx
from typing import Dict
from pydantic_pkgr import (
AptProvider,
BrewProvider,
EnvProvider,
BinProvider,
)
apt = APT_BINPROVIDER = AptProvider()
brew = BREW_BINPROVIDER = BrewProvider()
env = ENV_BINPROVIDER = EnvProvider()
@abx.hookimpl(tryfirst=True)
def get_BINPROVIDERS() -> Dict[str, BinProvider]:
return {
'apt': APT_BINPROVIDER,
'brew': BREW_BINPROVIDER,
'env': ENV_BINPROVIDER,
}

18
archivebox/vendor/abx-plugin-default-binproviders/pyproject.toml vendored Normal file
View File

@@ -0,0 +1,18 @@
[project]
name = "abx-plugin-default-binproviders"
version = "2024.10.24"
description = "Default BinProviders for ABX (apt, brew, env)"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"pydantic-pkgr>=0.5.4",
"abx-spec-pydantic-pkgr>=0.1.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_default_binproviders = "abx_plugin_default_binproviders"

0
archivebox/vendor/abx-plugin-favicon/README.md vendored Normal file
View File

29
archivebox/vendor/abx-plugin-favicon/abx_plugin_favicon/__init__.py vendored Normal file
View File

@@ -0,0 +1,29 @@
__label__ = 'Favicon'
__version__ = '2024.10.24'
__author__ = 'ArchiveBox'
__homepage__ = 'https://github.com/ArchiveBox/archivebox'
__dependencies__ = [
'abx>=0.1.0',
'abx-spec-config>=0.1.0',
'abx-plugin-curl-extractor>=2024.10.24',
]
import abx
@abx.hookimpl
def get_CONFIG():
from .config import FAVICON_CONFIG
return {
'FAVICON_CONFIG': FAVICON_CONFIG
}
# @abx.hookimpl
# def get_EXTRACTORS():
# from .extractors import FAVICON_EXTRACTOR
# return {
# 'favicon': FAVICON_EXTRACTOR,
# }

10
archivebox/vendor/abx-plugin-favicon/abx_plugin_favicon/config.py vendored Normal file
View File

@@ -0,0 +1,10 @@
from abx_spec_config.base_configset import BaseConfigSet
class FaviconConfig(BaseConfigSet):
SAVE_FAVICON: bool = True
FAVICON_PROVIDER: str = 'https://www.google.com/s2/favicons?domain={}'
FAVICON_CONFIG = FaviconConfig()

0
archivebox/extractors/favicon.py → archivebox/vendor/abx-plugin-favicon/abx_plugin_favicon/favicon.py vendored
View File

18
archivebox/vendor/abx-plugin-favicon/pyproject.toml vendored Normal file
View File

@@ -0,0 +1,18 @@
[project]
name = "abx-plugin-favicon"
version = "2024.10.28"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"abx-spec-config>=0.1.0",
"abx-plugin-curl>=2024.10.28",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_favicon = "abx_plugin_favicon"

0
archivebox/vendor/abx-plugin-git/README.md vendored Normal file
View File

29
archivebox/vendor/abx-plugin-git/abx_plugin_git/__init__.py vendored Normal file
View File

@@ -0,0 +1,29 @@
__package__ = 'abx_plugin_git'
__label__ = 'Git'
import abx
@abx.hookimpl
def get_CONFIG():
from .config import GIT_CONFIG
return {
'GIT_CONFIG': GIT_CONFIG
}
@abx.hookimpl
def get_BINARIES():
from .binaries import GIT_BINARY
return {
'git': GIT_BINARY,
}
@abx.hookimpl
def get_EXTRACTORS():
from .extractors import GIT_EXTRACTOR
return {
'git': GIT_EXTRACTOR,
}

18
archivebox/vendor/abx-plugin-git/abx_plugin_git/binaries.py vendored Normal file
View File

@@ -0,0 +1,18 @@
__package__ = 'abx_plugin_git'
from typing import List
from pydantic import InstanceOf
from pydantic_pkgr import BinProvider, BinName, Binary
from abx_plugin_default_binproviders import apt, brew, env
from .config import GIT_CONFIG
class GitBinary(Binary):
name: BinName = GIT_CONFIG.GIT_BINARY
binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
GIT_BINARY = GitBinary()

28
archivebox/vendor/abx-plugin-git/abx_plugin_git/config.py vendored Normal file
View File

@@ -0,0 +1,28 @@
__package__ = 'abx_plugin_git'
from typing import List
from pydantic import Field
from abx_spec_config.base_configset import BaseConfigSet
from archivebox.config.common import ARCHIVING_CONFIG
class GitConfig(BaseConfigSet):
SAVE_GIT: bool = True
GIT_DOMAINS: str = Field(default='github.com,bitbucket.org,gitlab.com,gist.github.com,codeberg.org,gitea.com,git.sr.ht')
GIT_BINARY: str = Field(default='git')
GIT_ARGS: List[str] = [
'--recursive',
]
GIT_EXTRA_ARGS: List[str] = []
GIT_TIMEOUT: int = Field(default=lambda: ARCHIVING_CONFIG.TIMEOUT)
GIT_CHECK_SSL_VALIDITY: bool = Field(default=lambda: ARCHIVING_CONFIG.CHECK_SSL_VALIDITY)
GIT_CONFIG = GitConfig()

15
archivebox/vendor/abx-plugin-git/abx_plugin_git/extractors.py vendored Normal file
View File

@@ -0,0 +1,15 @@
__package__ = 'abx_plugin_git'
# from pathlib import Path
# from .binaries import GIT_BINARY
# class GitExtractor(BaseExtractor):
# name: ExtractorName = 'git'
# binary: str = GIT_BINARY.name
# def get_output_path(self, snapshot) -> Path | None:
# return snapshot.as_link() / 'git'
# GIT_EXTRACTOR = GitExtractor()

4
archivebox/extractors/git.py → archivebox/vendor/abx-plugin-git/abx_plugin_git/git.py vendored
View File

@@ -16,8 +16,8 @@ from archivebox.misc.util import (
from ..logging_util import TimedProgress
from ..index.schema import Link, ArchiveResult, ArchiveOutput, ArchiveError
from archivebox.plugins_extractor.git.config import GIT_CONFIG
from archivebox.plugins_extractor.git.binaries import GIT_BINARY
from abx_plugin_git.config import GIT_CONFIG
from abx_plugin_git.binaries import GIT_BINARY
def get_output_path():

19
archivebox/vendor/abx-plugin-git/pyproject.toml vendored Normal file
View File

@@ -0,0 +1,19 @@
[project]
name = "abx-plugin-git"
version = "2024.10.28"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"abx-spec-config>=0.1.0",
"abx-spec-pydantic-pkgr>=0.1.0",
"abx-plugin-default-binproviders>=2024.10.24",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_git = "abx_plugin_git"

0
archivebox/vendor/abx-plugin-htmltotext/README.md vendored Normal file
View File

22
archivebox/vendor/abx-plugin-htmltotext/abx_plugin_htmltotext/__init__.py vendored Normal file
View File

@@ -0,0 +1,22 @@
__package__ = 'abx_plugin_htmltotext'
__label__ = 'HTML-to-Text'
import abx
@abx.hookimpl
def get_CONFIG():
from .config import HTMLTOTEXT_CONFIG
return {
'HTMLTOTEXT_CONFIG': HTMLTOTEXT_CONFIG
}
# @abx.hookimpl
# def get_EXTRACTORS():
# from .extractors import FAVICON_EXTRACTOR
# return {
# 'htmltotext': FAVICON_EXTRACTOR,
# }

8
archivebox/vendor/abx-plugin-htmltotext/abx_plugin_htmltotext/config.py vendored Normal file
View File

@@ -0,0 +1,8 @@
from abx_spec_config.base_configset import BaseConfigSet
class HtmltotextConfig(BaseConfigSet):
SAVE_HTMLTOTEXT: bool = True
HTMLTOTEXT_CONFIG = HtmltotextConfig()

0
archivebox/extractors/htmltotext.py → archivebox/vendor/abx-plugin-htmltotext/abx_plugin_htmltotext/htmltotext.py vendored
View File

17
archivebox/vendor/abx-plugin-htmltotext/pyproject.toml vendored Normal file
View File

@@ -0,0 +1,17 @@
[project]
name = "abx-plugin-htmltotext"
version = "2024.10.28"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"abx-spec-config>=0.1.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_htmltotext = "abx_plugin_htmltotext"

0
archivebox/vendor/abx-plugin-ldap-auth/README.md vendored Normal file
View File

54
archivebox/vendor/abx-plugin-ldap-auth/abx_plugin_ldap_auth/__init__.py vendored Normal file
View File

@@ -0,0 +1,54 @@
__package__ = 'abx_plugin_ldap_auth'
__label__ = 'LDAP'
__homepage__ = 'https://github.com/django-auth-ldap/django-auth-ldap'
import abx
@abx.hookimpl
def get_CONFIG():
from .config import LDAP_CONFIG
return {
'LDAP_CONFIG': LDAP_CONFIG
}
@abx.hookimpl
def get_BINARIES():
from .binaries import LDAP_BINARY
return {
'ldap': LDAP_BINARY,
}
def create_superuser_from_ldap_user(sender, user=None, ldap_user=None, **kwargs):
"""
Invoked after LDAP authenticates a user, but before they have a local User account created.
ArchiveBox requires staff/superuser status to view the admin at all, so we must create a user
+ set staff and superuser when LDAP authenticates a new person.
"""
from .config import LDAP_CONFIG
if user is None:
return # not authenticated at all
if not user.id and LDAP_CONFIG.LDAP_CREATE_SUPERUSER:
user.is_superuser = True # authenticated via LDAP, but user is not set up in DB yet
user.is_staff = True
print(f'[!] WARNING: Creating new user {user} based on LDAP user {ldap_user} (is_staff={user.is_staff}, is_superuser={user.is_superuser})')
@abx.hookimpl
def ready():
"""
Called at AppConfig.ready() time (settings + models are all loaded)
"""
from .config import LDAP_CONFIG
LDAP_CONFIG.validate()
if LDAP_CONFIG.LDAP_ENABLED:
# tell django-auth-ldap to call our function when a user is authenticated via LDAP
import django_auth_ldap.backend
django_auth_ldap.backend.populate_user.connect(create_superuser_from_ldap_user)

67
archivebox/vendor/abx-plugin-ldap-auth/abx_plugin_ldap_auth/binaries.py vendored Normal file
View File

@@ -0,0 +1,67 @@
__package__ = 'abx_plugin_ldap_auth'
import inspect
from typing import List
from pathlib import Path
from pydantic import InstanceOf
from pydantic_pkgr import BinaryOverrides, SemVer, Binary, BinProvider
from abx_plugin_default_binproviders import apt
from abx_plugin_pip.binproviders import SYS_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, LIB_PIP_BINPROVIDER, VENV_SITE_PACKAGES, LIB_SITE_PACKAGES, USER_SITE_PACKAGES, SYS_SITE_PACKAGES
from .config import get_ldap_lib
def get_LDAP_LIB_path(paths=()):
LDAP_LIB = get_ldap_lib()[0]
if not LDAP_LIB:
return None
# check that LDAP_LIB path is in one of the specified site packages dirs
lib_path = Path(inspect.getfile(LDAP_LIB))
if not paths:
return lib_path
for site_packges_dir in paths:
if str(lib_path.parent.parent.resolve()) == str(Path(site_packges_dir).resolve()):
return lib_path
return None
def get_LDAP_LIB_version():
LDAP_LIB = get_ldap_lib()[0]
return LDAP_LIB and SemVer(LDAP_LIB.__version__)
class LdapBinary(Binary):
name: str = 'ldap'
description: str = 'LDAP Authentication'
binproviders_supported: List[InstanceOf[BinProvider]] = [VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, LIB_PIP_BINPROVIDER, apt]
overrides: BinaryOverrides = {
LIB_PIP_BINPROVIDER.name: {
"abspath": lambda: get_LDAP_LIB_path(LIB_SITE_PACKAGES),
"version": lambda: get_LDAP_LIB_version(),
"packages": ['python-ldap>=3.4.3', 'django-auth-ldap>=4.1.0'],
},
VENV_PIP_BINPROVIDER.name: {
"abspath": lambda: get_LDAP_LIB_path(VENV_SITE_PACKAGES),
"version": lambda: get_LDAP_LIB_version(),
"packages": ['python-ldap>=3.4.3', 'django-auth-ldap>=4.1.0'],
},
SYS_PIP_BINPROVIDER.name: {
"abspath": lambda: get_LDAP_LIB_path((*USER_SITE_PACKAGES, *SYS_SITE_PACKAGES)),
"version": lambda: get_LDAP_LIB_version(),
"packages": ['python-ldap>=3.4.3', 'django-auth-ldap>=4.1.0'],
},
apt.name: {
"abspath": lambda: get_LDAP_LIB_path(),
"version": lambda: get_LDAP_LIB_version(),
"packages": ['libssl-dev', 'libldap2-dev', 'libsasl2-dev', 'python3-ldap', 'python3-msgpack', 'python3-mutagen'],
},
}
LDAP_BINARY = LdapBinary()

122
archivebox/vendor/abx-plugin-ldap-auth/abx_plugin_ldap_auth/config.py vendored Normal file
View File

@@ -0,0 +1,122 @@
__package__ = 'abx_plugin_ldap_auth'
import sys
from typing import Dict, List, Optional
from pydantic import Field, computed_field
from abx_spec_config.base_configset import BaseConfigSet
LDAP_LIB = None
LDAP_SEARCH = None
def get_ldap_lib(extra_paths=()):
global LDAP_LIB, LDAP_SEARCH
if LDAP_LIB and LDAP_SEARCH:
return LDAP_LIB, LDAP_SEARCH
try:
for path in extra_paths:
if path not in sys.path:
sys.path.append(path)
import ldap
from django_auth_ldap.config import LDAPSearch
LDAP_LIB, LDAP_SEARCH = ldap, LDAPSearch
except ImportError:
pass
return LDAP_LIB, LDAP_SEARCH
###################### Config ##########################
class LdapConfig(BaseConfigSet):
"""
LDAP Config gets imported by core/settings.py very early during startup.
It needs to be in a separate file from apps.py so that it can be imported
during settings.py initialization before the apps are loaded.
"""
LDAP_ENABLED: bool = Field(default=False, alias='LDAP')
LDAP_SERVER_URI: str = Field(default=None)
LDAP_BIND_DN: str = Field(default=None)
LDAP_BIND_PASSWORD: str = Field(default=None)
LDAP_USER_BASE: str = Field(default=None)
LDAP_USER_FILTER: str = Field(default=None)
LDAP_CREATE_SUPERUSER: bool = Field(default=False)
LDAP_USERNAME_ATTR: str = Field(default='username')
LDAP_FIRSTNAME_ATTR: str = Field(default='first_name')
LDAP_LASTNAME_ATTR: str = Field(default='last_name')
LDAP_EMAIL_ATTR: str = Field(default='email')
def validate(self):
if self.LDAP_ENABLED:
LDAP_LIB, _LDAPSearch = get_ldap_lib()
# Check that LDAP libraries are installed
if LDAP_LIB is None:
sys.stderr.write('[X] Error: LDAP Authentication is enabled but LDAP libraries are not installed. You may need to run: pip install archivebox[ldap]\n')
# dont hard exit here. in case the user is just running "archivebox version" or "archivebox help", we still want those to work despite broken ldap
# sys.exit(1)
self.update_in_place(LDAP_ENABLED=False)
# Check that all required LDAP config options are set
if self.LDAP_CONFIG_IS_SET:
missing_config_options = [
key for key, value in self.model_dump().items()
if value is None and key != 'LDAP_ENABLED'
]
sys.stderr.write('[X] Error: LDAP_* config options must all be set if LDAP_ENABLED=True\n')
sys.stderr.write(f' Missing: {", ".join(missing_config_options)}\n')
self.update_in_place(LDAP_ENABLED=False)
return self
@computed_field
@property
def LDAP_CONFIG_IS_SET(self) -> bool:
"""Check that all required LDAP config options are set"""
if self.LDAP_ENABLED:
LDAP_LIB, _LDAPSearch = get_ldap_lib()
return bool(LDAP_LIB) and self.LDAP_ENABLED and bool(
self.LDAP_SERVER_URI
and self.LDAP_BIND_DN
and self.LDAP_BIND_PASSWORD
and self.LDAP_USER_BASE
and self.LDAP_USER_FILTER
)
return False
@computed_field
@property
def LDAP_USER_ATTR_MAP(self) -> Dict[str, str]:
return {
'username': self.LDAP_USERNAME_ATTR,
'first_name': self.LDAP_FIRSTNAME_ATTR,
'last_name': self.LDAP_LASTNAME_ATTR,
'email': self.LDAP_EMAIL_ATTR,
}
@computed_field
@property
def AUTHENTICATION_BACKENDS(self) -> List[str]:
if self.LDAP_ENABLED:
return [
'django.contrib.auth.backends.ModelBackend',
'django_auth_ldap.backend.LDAPBackend',
]
return []
@computed_field
@property
def AUTH_LDAP_USER_SEARCH(self) -> Optional[object]:
if self.LDAP_ENABLED:
LDAP_LIB, LDAPSearch = get_ldap_lib()
return self.LDAP_USER_FILTER and LDAPSearch(
self.LDAP_USER_BASE,
LDAP_LIB.SCOPE_SUBTREE, # type: ignore
'(&(' + self.LDAP_USERNAME_ATTR + '=%(user)s)' + self.LDAP_USER_FILTER + ')',
)
return None
LDAP_CONFIG = LdapConfig()

20
archivebox/vendor/abx-plugin-ldap-auth/pyproject.toml vendored Normal file
View File

@@ -0,0 +1,20 @@
[project]
name = "abx-plugin-ldap-auth"
version = "2024.10.28"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"abx-spec-config>=0.1.0",
"abx-spec-django>=0.1.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_ldap_auth = "abx_plugin_ldap_auth"

0
archivebox/vendor/abx-plugin-mercury/README.md vendored Normal file
View File

29
archivebox/vendor/abx-plugin-mercury/abx_plugin_mercury/__init__.py vendored Normal file
View File

@@ -0,0 +1,29 @@
__package__ = 'abx_plugin_mercury'
__label__ = 'Postlight Parser'
__homepage__ = 'https://github.com/postlight/mercury-parser'
import abx
@abx.hookimpl
def get_CONFIG():
from .config import MERCURY_CONFIG
return {
'MERCURY_CONFIG': MERCURY_CONFIG
}
@abx.hookimpl
def get_BINARIES():
from .binaries import MERCURY_BINARY
return {
'mercury': MERCURY_BINARY,
}
@abx.hookimpl
def get_EXTRACTORS():
from .extractors import MERCURY_EXTRACTOR
return {
'mercury': MERCURY_EXTRACTOR,
}

32
archivebox/vendor/abx-plugin-mercury/abx_plugin_mercury/binaries.py vendored Normal file
View File

@@ -0,0 +1,32 @@
__package__ = 'abx_plugin_mercury'
from typing import List
from pydantic import InstanceOf
from pydantic_pkgr import BinProvider, BinName, BinaryOverrides, bin_abspath, Binary
from abx_plugin_default_binproviders import env
from abx_plugin_npm.binproviders import SYS_NPM_BINPROVIDER, LIB_NPM_BINPROVIDER
from .config import MERCURY_CONFIG
class MercuryBinary(Binary):
name: BinName = MERCURY_CONFIG.MERCURY_BINARY
binproviders_supported: List[InstanceOf[BinProvider]] = [LIB_NPM_BINPROVIDER, SYS_NPM_BINPROVIDER, env]
overrides: BinaryOverrides = {
LIB_NPM_BINPROVIDER.name: {
'packages': ['@postlight/parser@^2.2.3'],
},
SYS_NPM_BINPROVIDER.name: {
'packages': ['@postlight/parser@^2.2.3'],
'install': lambda: None, # never try to install things into global prefix
},
env.name: {
'version': lambda: '999.999.999' if bin_abspath('postlight-parser', PATH=env.PATH) else None,
},
}
MERCURY_BINARY = MercuryBinary()

31
archivebox/vendor/abx-plugin-mercury/abx_plugin_mercury/config.py vendored Normal file
View File

@@ -0,0 +1,31 @@
__package__ = 'abx_plugin_mercury'
from typing import List, Optional
from pathlib import Path
from pydantic import Field
from abx_spec_config.base_configset import BaseConfigSet
from archivebox.config.common import ARCHIVING_CONFIG, STORAGE_CONFIG
class MercuryConfig(BaseConfigSet):
SAVE_MERCURY: bool = Field(default=True, alias='USE_MERCURY')
MERCURY_BINARY: str = Field(default='postlight-parser')
MERCURY_EXTRA_ARGS: List[str] = []
SAVE_MERCURY_REQUISITES: bool = Field(default=True)
MERCURY_RESTRICT_FILE_NAMES: str = Field(default=lambda: STORAGE_CONFIG.RESTRICT_FILE_NAMES)
MERCURY_TIMEOUT: int = Field(default=lambda: ARCHIVING_CONFIG.TIMEOUT)
MERCURY_CHECK_SSL_VALIDITY: bool = Field(default=lambda: ARCHIVING_CONFIG.CHECK_SSL_VALIDITY)
MERCURY_USER_AGENT: str = Field(default=lambda: ARCHIVING_CONFIG.USER_AGENT)
MERCURY_COOKIES_FILE: Optional[Path] = Field(default=lambda: ARCHIVING_CONFIG.COOKIES_FILE)
MERCURY_CONFIG = MercuryConfig()

17
archivebox/vendor/abx-plugin-mercury/abx_plugin_mercury/extractors.py vendored Normal file
View File

@@ -0,0 +1,17 @@
__package__ = 'abx_plugin_mercury'
# from pathlib import Path
# from .binaries import MERCURY_BINARY
# class MercuryExtractor(BaseExtractor):
# name: ExtractorName = 'mercury'
# binary: str = MERCURY_BINARY.name
# def get_output_path(self, snapshot) -> Path | None:
# return snapshot.link_dir / 'mercury' / 'content.html'
# MERCURY_EXTRACTOR = MercuryExtractor()

0
archivebox/extractors/mercury.py → archivebox/vendor/abx-plugin-mercury/abx_plugin_mercury/mercury.py vendored
View File

17
archivebox/vendor/abx-plugin-mercury/pyproject.toml vendored Normal file
View File

@@ -0,0 +1,17 @@
[project]
name = "abx-plugin-mercury"
version = "2024.10.28"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"abx-spec-config>=0.1.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_mercury = "abx_plugin_mercury"

0
archivebox/vendor/abx-plugin-npm/README.md vendored Normal file
View File

32
archivebox/vendor/abx-plugin-npm/abx_plugin_npm/__init__.py vendored Normal file
View File

@@ -0,0 +1,32 @@
__label__ = 'NPM'
__author__ = 'ArchiveBox'
__homepage__ = 'https://www.npmjs.com/'
import abx
@abx.hookimpl
def get_CONFIG():
from .config import NPM_CONFIG
return {
'NPM_CONFIG': NPM_CONFIG,
}
@abx.hookimpl
def get_BINARIES():
from .binaries import NODE_BINARY, NPM_BINARY, NPX_BINARY
return {
'node': NODE_BINARY,
'npm': NPM_BINARY,
'npx': NPX_BINARY,
}
@abx.hookimpl
def get_BINPROVIDERS():
from .binproviders import LIB_NPM_BINPROVIDER, SYS_NPM_BINPROVIDER
return {
'sys_npm': SYS_NPM_BINPROVIDER,
'lib_npm': LIB_NPM_BINPROVIDER,
}

53
archivebox/vendor/abx-plugin-npm/abx_plugin_npm/binaries.py vendored Normal file
View File

@@ -0,0 +1,53 @@
__package__ = 'plugins_pkg.npm'
from typing import List
from pydantic import InstanceOf
from benedict import benedict
from pydantic_pkgr import BinProvider, Binary, BinName, BinaryOverrides
from abx_plugin_default_binproviders import get_BINPROVIDERS
DEFAULT_BINPROVIDERS = benedict(get_BINPROVIDERS())
env = DEFAULT_BINPROVIDERS.env
apt = DEFAULT_BINPROVIDERS.apt
brew = DEFAULT_BINPROVIDERS.brew
class NodeBinary(Binary):
name: BinName = 'node'
binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
overrides: BinaryOverrides = {
apt.name: {'packages': ['nodejs']},
}
NODE_BINARY = NodeBinary()
class NpmBinary(Binary):
name: BinName = 'npm'
binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
overrides: BinaryOverrides = {
apt.name: {'packages': ['npm']}, # already installed when nodejs is installed
brew.name: {'install': lambda: None}, # already installed when nodejs is installed
}
NPM_BINARY = NpmBinary()
class NpxBinary(Binary):
name: BinName = 'npx'
binproviders_supported: List[InstanceOf[BinProvider]] = [apt, brew, env]
overrides: BinaryOverrides = {
apt.name: {'install': lambda: None}, # already installed when nodejs is installed
brew.name: {'install': lambda: None}, # already installed when nodejs is installed
}
NPX_BINARY = NpxBinary()

38
archivebox/vendor/abx-plugin-npm/abx_plugin_npm/binproviders.py vendored Normal file
View File

@@ -0,0 +1,38 @@
import os
from pathlib import Path
from typing import Optional
from pydantic_pkgr import NpmProvider, PATHStr, BinProviderName
import abx
DEFAULT_LIB_NPM_DIR = Path('/usr/local/share/abx/npm')
OLD_NODE_BIN_PATH = Path(os.getcwd()) / 'node_modules' / '.bin'
NEW_NODE_BIN_PATH = DEFAULT_LIB_NPM_DIR / 'node_modules' / '.bin'
class SystemNpmBinProvider(NpmProvider):
name: BinProviderName = "sys_npm"
npm_prefix: Optional[Path] = None
class LibNpmBinProvider(NpmProvider):
name: BinProviderName = "lib_npm"
PATH: PATHStr = f'{NEW_NODE_BIN_PATH}:{OLD_NODE_BIN_PATH}'
npm_prefix: Optional[Path] = DEFAULT_LIB_NPM_DIR
def setup(self) -> None:
# update paths from config at runtime
LIB_DIR = abx.pm.hook.get_LIB_DIR()
self.npm_prefix = LIB_DIR / 'npm'
self.PATH = f'{LIB_DIR / "npm" / "node_modules" / ".bin"}:{NEW_NODE_BIN_PATH}:{OLD_NODE_BIN_PATH}'
super().setup()
SYS_NPM_BINPROVIDER = SystemNpmBinProvider()
LIB_NPM_BINPROVIDER = LibNpmBinProvider()
npm = LIB_NPM_BINPROVIDER

17
archivebox/vendor/abx-plugin-npm/abx_plugin_npm/config.py vendored Normal file
View File

@@ -0,0 +1,17 @@
from abx_spec_config import BaseConfigSet
###################### Config ##########################
class NpmDependencyConfigs(BaseConfigSet):
# USE_NPM: bool = True
# NPM_BINARY: str = Field(default='npm')
# NPM_ARGS: Optional[List[str]] = Field(default=None)
# NPM_EXTRA_ARGS: List[str] = []
# NPM_DEFAULT_ARGS: List[str] = []
pass
NPM_CONFIG = NpmDependencyConfigs()

20
archivebox/vendor/abx-plugin-npm/pyproject.toml vendored Normal file
View File

@@ -0,0 +1,20 @@
[project]
name = "abx-plugin-npm"
version = "2024.10.24"
description = "NPM binary provider plugin for ABX"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"pydantic-pkgr>=0.5.4",
"abx-spec-pydantic-pkgr>=0.1.0",
"abx-spec-config>=0.1.0",
"abx-plugin-default-binproviders>=2024.10.24",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_npm = "abx_plugin_npm"

0
archivebox/vendor/abx-plugin-pip/README.md vendored Normal file
View File

1
archivebox/vendor/abx-plugin-pip/abx_plugin_pip/.plugin_order vendored Normal file
View File

@@ -0,0 +1 @@
0

36
archivebox/vendor/abx-plugin-pip/abx_plugin_pip/__init__.py vendored Normal file
View File

@@ -0,0 +1,36 @@
__package__ = 'abx_plugin_pip'
__label__ = 'PIP'
import abx
@abx.hookimpl
def get_CONFIG():
from .config import PIP_CONFIG
return {
'PIP_CONFIG': PIP_CONFIG
}
@abx.hookimpl(tryfirst=True)
def get_BINARIES():
from .binaries import ARCHIVEBOX_BINARY, PYTHON_BINARY, DJANGO_BINARY, SQLITE_BINARY, PIP_BINARY, PIPX_BINARY
return {
'archivebox': ARCHIVEBOX_BINARY,
'python': PYTHON_BINARY,
'django': DJANGO_BINARY,
'sqlite': SQLITE_BINARY,
'pip': PIP_BINARY,
'pipx': PIPX_BINARY,
}
@abx.hookimpl
def get_BINPROVIDERS():
from .binproviders import SYS_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, LIB_PIP_BINPROVIDER
return {
'sys_pip': SYS_PIP_BINPROVIDER,
'venv_pip': VENV_PIP_BINPROVIDER,
'lib_pip': LIB_PIP_BINPROVIDER,
}

162
archivebox/vendor/abx-plugin-pip/abx_plugin_pip/binaries.py vendored Normal file
View File

@@ -0,0 +1,162 @@
__package__ = 'abx_plugin_pip'
import sys
from pathlib import Path
from typing import List
from pydantic import InstanceOf, Field, model_validator
import django
import django.db.backends.sqlite3.base
from django.db.backends.sqlite3.base import Database as django_sqlite3 # type: ignore[import-type]
from pydantic_pkgr import BinProvider, Binary, BinName, BinaryOverrides, SemVer
from .binproviders import LIB_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, env, apt, brew
###################### Config ##########################
def get_archivebox_version():
try:
from archivebox import VERSION
return VERSION
except Exception:
return None
class ArchiveboxBinary(Binary):
name: BinName = 'archivebox'
binproviders_supported: List[InstanceOf[BinProvider]] = [VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env]
overrides: BinaryOverrides = {
VENV_PIP_BINPROVIDER.name: {'packages': [], 'version': get_archivebox_version},
SYS_PIP_BINPROVIDER.name: {'packages': [], 'version': get_archivebox_version},
apt.name: {'packages': [], 'version': get_archivebox_version},
brew.name: {'packages': [], 'version': get_archivebox_version},
}
# @validate_call
def install(self, **kwargs):
return self.load() # obviously it's already installed if we are running this ;)
# @validate_call
def load_or_install(self, **kwargs):
return self.load() # obviously it's already installed if we are running this ;)
ARCHIVEBOX_BINARY = ArchiveboxBinary()
class PythonBinary(Binary):
name: BinName = 'python'
binproviders_supported: List[InstanceOf[BinProvider]] = [VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env]
overrides: BinaryOverrides = {
SYS_PIP_BINPROVIDER.name: {
'abspath': sys.executable,
'version': '{}.{}.{}'.format(*sys.version_info[:3]),
},
}
# @validate_call
def install(self, **kwargs):
return self.load() # obviously it's already installed if we are running this ;)
# @validate_call
def load_or_install(self, **kwargs):
return self.load() # obviously it's already installed if we are running this ;)
PYTHON_BINARY = PythonBinary()
LOADED_SQLITE_PATH = Path(django.db.backends.sqlite3.base.__file__)
LOADED_SQLITE_VERSION = SemVer(django_sqlite3.version)
LOADED_SQLITE_FROM_VENV = str(LOADED_SQLITE_PATH.absolute().resolve()).startswith(str(VENV_PIP_BINPROVIDER.pip_venv.absolute().resolve()))
class SqliteBinary(Binary):
name: BinName = 'sqlite'
binproviders_supported: List[InstanceOf[BinProvider]] = Field(default=[VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER])
overrides: BinaryOverrides = {
VENV_PIP_BINPROVIDER.name: {
"abspath": LOADED_SQLITE_PATH if LOADED_SQLITE_FROM_VENV else None,
"version": LOADED_SQLITE_VERSION if LOADED_SQLITE_FROM_VENV else None,
},
SYS_PIP_BINPROVIDER.name: {
"abspath": LOADED_SQLITE_PATH if not LOADED_SQLITE_FROM_VENV else None,
"version": LOADED_SQLITE_VERSION if not LOADED_SQLITE_FROM_VENV else None,
},
}
@model_validator(mode='after')
def validate_json_extension_is_available(self):
# Check to make sure JSON extension is available in our Sqlite3 instance
try:
cursor = django_sqlite3.connect(':memory:').cursor()
cursor.execute('SELECT JSON(\'{"a": "b"}\')')
except django_sqlite3.OperationalError as exc:
print(f'[red][X] Your SQLite3 version is missing the required JSON1 extension: {exc}[/red]')
print(
'[violet]Hint:[/violet] Upgrade your Python version or install the extension manually:\n' +
' https://code.djangoproject.com/wiki/JSON1Extension\n'
)
return self
# @validate_call
def install(self, **kwargs):
return self.load() # obviously it's already installed if we are running this ;)
# @validate_call
def load_or_install(self, **kwargs):
return self.load() # obviously it's already installed if we are running this ;)
SQLITE_BINARY = SqliteBinary()
LOADED_DJANGO_PATH = Path(django.__file__)
LOADED_DJANGO_VERSION = SemVer(django.VERSION[:3])
LOADED_DJANGO_FROM_VENV = str(LOADED_DJANGO_PATH.absolute().resolve()).startswith(str(VENV_PIP_BINPROVIDER.pip_venv and VENV_PIP_BINPROVIDER.pip_venv.absolute().resolve()))
class DjangoBinary(Binary):
name: BinName = 'django'
binproviders_supported: List[InstanceOf[BinProvider]] = Field(default=[VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER])
overrides: BinaryOverrides = {
VENV_PIP_BINPROVIDER.name: {
"abspath": LOADED_DJANGO_PATH if LOADED_DJANGO_FROM_VENV else None,
"version": LOADED_DJANGO_VERSION if LOADED_DJANGO_FROM_VENV else None,
},
SYS_PIP_BINPROVIDER.name: {
"abspath": LOADED_DJANGO_PATH if not LOADED_DJANGO_FROM_VENV else None,
"version": LOADED_DJANGO_VERSION if not LOADED_DJANGO_FROM_VENV else None,
},
}
# @validate_call
def install(self, **kwargs):
return self.load() # obviously it's already installed if we are running this ;)
# @validate_call
def load_or_install(self, **kwargs):
return self.load() # obviously it's already installed if we are running this ;)
DJANGO_BINARY = DjangoBinary()
class PipBinary(Binary):
name: BinName = "pip"
binproviders_supported: List[InstanceOf[BinProvider]] = [LIB_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env]
# @validate_call
def install(self, **kwargs):
return self.load() # obviously it's already installed if we are running this ;)
# @validate_call
def load_or_install(self, **kwargs):
return self.load() # obviously it's already installed if we are running this ;)
PIP_BINARY = PipBinary()
class PipxBinary(Binary):
name: BinName = "pipx"
binproviders_supported: List[InstanceOf[BinProvider]] = [LIB_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, apt, brew, env]
PIPX_BINARY = PipxBinary()

91
archivebox/vendor/abx-plugin-pip/abx_plugin_pip/binproviders.py vendored Normal file
View File

@@ -0,0 +1,91 @@
import os
import sys
import site
from pathlib import Path
from typing import Optional
from benedict import benedict
from pydantic_pkgr import PipProvider, BinName, BinProviderName
import abx
from abx_plugin_default_binproviders import get_BINPROVIDERS
DEFAULT_BINPROVIDERS = benedict(get_BINPROVIDERS())
env = DEFAULT_BINPROVIDERS.env
apt = DEFAULT_BINPROVIDERS.apt
brew = DEFAULT_BINPROVIDERS.brew
###################### Config ##########################
class SystemPipBinProvider(PipProvider):
name: BinProviderName = "sys_pip"
INSTALLER_BIN: BinName = "pip"
pip_venv: Optional[Path] = None # global pip scope
def on_install(self, bin_name: str, **kwargs):
# never modify system pip packages
return 'refusing to install packages globally with system pip, use a venv instead'
class SystemPipxBinProvider(PipProvider):
name: BinProviderName = "pipx"
INSTALLER_BIN: BinName = "pipx"
pip_venv: Optional[Path] = None # global pipx scope
IS_INSIDE_VENV = sys.prefix != sys.base_prefix
class VenvPipBinProvider(PipProvider):
name: BinProviderName = "venv_pip"
INSTALLER_BIN: BinName = "pip"
pip_venv: Optional[Path] = Path(sys.prefix if IS_INSIDE_VENV else os.environ.get("VIRTUAL_ENV", '/tmp/NotInsideAVenv/lib'))
def setup(self):
"""never attempt to create a venv here, this is just used to detect if we are inside an existing one"""
return None
class LibPipBinProvider(PipProvider):
name: BinProviderName = "lib_pip"
INSTALLER_BIN: BinName = "pip"
pip_venv: Optional[Path] = Path('/usr/local/share/abx/pip/venv')
def setup(self) -> None:
# update venv path to match most up-to-date LIB_DIR based on runtime config
LIB_DIR = abx.pm.hook.get_LIB_DIR()
self.pip_venv = LIB_DIR / 'pip' / 'venv'
super().setup()
SYS_PIP_BINPROVIDER = SystemPipBinProvider()
PIPX_PIP_BINPROVIDER = SystemPipxBinProvider()
VENV_PIP_BINPROVIDER = VenvPipBinProvider()
LIB_PIP_BINPROVIDER = LibPipBinProvider()
pip = LIB_PIP_BINPROVIDER
# ensure python libraries are importable from these locations (if archivebox wasnt executed from one of these then they wont already be in sys.path)
assert VENV_PIP_BINPROVIDER.pip_venv is not None
assert LIB_PIP_BINPROVIDER.pip_venv is not None
major, minor, patch = sys.version_info[:3]
site_packages_dir = f'lib/python{major}.{minor}/site-packages'
LIB_SITE_PACKAGES = (LIB_PIP_BINPROVIDER.pip_venv / site_packages_dir,)
VENV_SITE_PACKAGES = (VENV_PIP_BINPROVIDER.pip_venv / site_packages_dir,)
USER_SITE_PACKAGES = site.getusersitepackages()
SYS_SITE_PACKAGES = site.getsitepackages()
ALL_SITE_PACKAGES = (
*LIB_SITE_PACKAGES,
*VENV_SITE_PACKAGES,
*USER_SITE_PACKAGES,
*SYS_SITE_PACKAGES,
)
for site_packages_dir in ALL_SITE_PACKAGES:
if site_packages_dir not in sys.path:
sys.path.append(str(site_packages_dir))

16
archivebox/vendor/abx-plugin-pip/abx_plugin_pip/config.py vendored Normal file
View File

@@ -0,0 +1,16 @@
__package__ = 'pip'
from typing import List, Optional
from pydantic import Field
from abx_spec_config.base_configset import BaseConfigSet
class PipDependencyConfigs(BaseConfigSet):
USE_PIP: bool = True
PIP_BINARY: str = Field(default='pip')
PIP_ARGS: Optional[List[str]] = Field(default=None)
PIP_EXTRA_ARGS: List[str] = []
PIP_DEFAULT_ARGS: List[str] = []
PIP_CONFIG = PipDependencyConfigs()

22
archivebox/vendor/abx-plugin-pip/pyproject.toml vendored Normal file
View File

@@ -0,0 +1,22 @@
[project]
name = "abx-plugin-pip"
version = "2024.10.24"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"pydantic-pkgr>=0.5.4",
"abx-spec-config>=0.1.0",
"abx-spec-pydantic-pkgr>=0.1.0",
"abx-plugin-default-binproviders>=2024.10.24",
"django>=5.0.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_pip = "abx_plugin_pip"

0
archivebox/vendor/abx-plugin-playwright/README.md vendored Normal file
View File

28
archivebox/vendor/abx-plugin-playwright/abx_plugin_playwright/__init__.py vendored Normal file
View File

@@ -0,0 +1,28 @@
__label__ = 'Playwright'
__homepage__ = 'https://github.com/microsoft/playwright-python'
import abx
@abx.hookimpl
def get_CONFIG():
from .config import PLAYWRIGHT_CONFIG
return {
'PLAYWRIGHT_CONFIG': PLAYWRIGHT_CONFIG
}
@abx.hookimpl
def get_BINARIES():
from .binaries import PLAYWRIGHT_BINARY
return {
'playwright': PLAYWRIGHT_BINARY,
}
@abx.hookimpl
def get_BINPROVIDERS():
from .binproviders import PLAYWRIGHT_BINPROVIDER
return {
'playwright': PLAYWRIGHT_BINPROVIDER,
}

21
archivebox/vendor/abx-plugin-playwright/abx_plugin_playwright/binaries.py vendored Normal file
View File

@@ -0,0 +1,21 @@
__package__ = 'abx_plugin_playwright'
from typing import List
from pydantic import InstanceOf
from pydantic_pkgr import BinName, BinProvider, Binary
from abx_plugin_pip.binproviders import LIB_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER
from abx_plugin_default_binproviders import env
from .config import PLAYWRIGHT_CONFIG
class PlaywrightBinary(Binary):
name: BinName = PLAYWRIGHT_CONFIG.PLAYWRIGHT_BINARY
binproviders_supported: List[InstanceOf[BinProvider]] = [LIB_PIP_BINPROVIDER, VENV_PIP_BINPROVIDER, SYS_PIP_BINPROVIDER, env]
PLAYWRIGHT_BINARY = PlaywrightBinary()

163
archivebox/vendor/abx-plugin-playwright/abx_plugin_playwright/binproviders.py vendored Normal file
View File

@@ -0,0 +1,163 @@
__package__ = 'abx_plugin_playwright'
import os
import shutil
import platform
from pathlib import Path
from typing import List, Optional, Dict, ClassVar
from pydantic import computed_field, Field
from pydantic_pkgr import (
BinName,
BinProvider,
BinProviderName,
BinProviderOverrides,
InstallArgs,
PATHStr,
HostBinPath,
bin_abspath,
OPERATING_SYSTEM,
DEFAULT_ENV_PATH,
)
import abx
from .binaries import PLAYWRIGHT_BINARY
MACOS_PLAYWRIGHT_CACHE_DIR: Path = Path("~/Library/Caches/ms-playwright")
LINUX_PLAYWRIGHT_CACHE_DIR: Path = Path("~/.cache/ms-playwright")
class PlaywrightBinProvider(BinProvider):
name: BinProviderName = "playwright"
INSTALLER_BIN: BinName = PLAYWRIGHT_BINARY.name
PATH: PATHStr = f"{Path('/usr/share/abx') / 'bin'}:{DEFAULT_ENV_PATH}"
playwright_browsers_dir: Path = (
MACOS_PLAYWRIGHT_CACHE_DIR.expanduser()
if OPERATING_SYSTEM == "darwin" else
LINUX_PLAYWRIGHT_CACHE_DIR.expanduser()
)
playwright_install_args: List[str] = ["install"]
packages_handler: BinProviderOverrides = Field(default={
"chrome": ["chromium"],
}, exclude=True)
_browser_abspaths: ClassVar[Dict[str, HostBinPath]] = {}
@computed_field
@property
def INSTALLER_BIN_ABSPATH(self) -> HostBinPath | None:
try:
return PLAYWRIGHT_BINARY.load().abspath
except Exception as e:
return None
def setup(self) -> None:
# update paths from config at runtime
LIB_DIR = abx.pm.hook.get_LIB_DIR()
self.PATH = f"{LIB_DIR / 'bin'}:{DEFAULT_ENV_PATH}"
assert shutil.which('pip'), "Pip bin provider not initialized"
if self.playwright_browsers_dir:
self.playwright_browsers_dir.mkdir(parents=True, exist_ok=True)
def installed_browser_bins(self, browser_name: str = "*") -> List[Path]:
if browser_name == 'chrome':
browser_name = 'chromium'
# if on macOS, browser binary is inside a .app, otherwise it's just a plain binary
if platform.system().lower() == "darwin":
# ~/Library/caches/ms-playwright/chromium-1097/chrome-mac/Chromium.app/Contents/MacOS/Chromium
return sorted(
self.playwright_browsers_dir.glob(
f"{browser_name}-*/*-mac*/*.app/Contents/MacOS/*"
)
)
# ~/Library/caches/ms-playwright/chromium-1097/chrome-linux/chromium
paths = []
for path in sorted(self.playwright_browsers_dir.glob(f"{browser_name}-*/*-linux/*")):
if 'xdg-settings' in str(path):
continue
if 'ffmpeg' in str(path):
continue
if '/chrom' in str(path) and 'chrom' in path.name.lower():
paths.append(path)
return paths
def default_abspath_handler(self, bin_name: BinName, **context) -> Optional[HostBinPath]:
assert bin_name == "chrome", "Only chrome is supported using the @puppeteer/browsers install method currently."
# already loaded, return abspath from cache
if bin_name in self._browser_abspaths:
return self._browser_abspaths[bin_name]
# first time loading, find browser in self.playwright_browsers_dir by searching filesystem for installed binaries
matching_bins = [abspath for abspath in self.installed_browser_bins() if bin_name in str(abspath)]
if matching_bins:
newest_bin = matching_bins[-1] # already sorted alphabetically, last should theoretically be highest version number
self._browser_abspaths[bin_name] = newest_bin
return self._browser_abspaths[bin_name]
# playwright sometimes installs google-chrome-stable via apt into system $PATH, check there as well
abspath = bin_abspath('google-chrome-stable', PATH=env.PATH)
if abspath:
self._browser_abspaths[bin_name] = abspath
return self._browser_abspaths[bin_name]
return None
def default_install_handler(self, bin_name: str, packages: Optional[InstallArgs] = None, **context) -> str:
"""playwright install chrome"""
self.setup()
assert bin_name == "chrome", "Only chrome is supported using the playwright install method currently."
if not self.INSTALLER_BIN_ABSPATH:
raise Exception(
f"{self.__class__.__name__} install method is not available on this host ({self.INSTALLER_BIN} not found in $PATH)"
)
packages = packages or self.get_packages(bin_name)
# print(f'[*] {self.__class__.__name__}: Installing {bin_name}: {self.INSTALLER_BIN_ABSPATH} install {packages}')
# playwright install-deps (to install system dependencies like fonts, graphics libraries, etc.)
if platform.system().lower() != 'darwin':
# libglib2.0-0, libnss3, libnspr4, libdbus-1-3, libatk1.0-0, libatk-bridge2.0-0, libcups2, libdrm2, libxcb1, libxkbcommon0, libatspi2.0-0, libx11-6, libxcomposite1, libxdamage1, libxext6, libxfixes3, libxrandr2, libgbm1, libcairo2, libasound2
proc = self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=['install-deps'])
if proc.returncode != 0:
print(proc.stdout.strip())
print(proc.stderr.strip())
proc = self.exec(bin_name=self.INSTALLER_BIN_ABSPATH, cmd=['install', *packages])
if proc.returncode != 0:
print(proc.stdout.strip())
print(proc.stderr.strip())
raise Exception(f"{self.__class__.__name__}: install got returncode {proc.returncode} while installing {packages}: {packages} PACKAGES={packages}")
# chrome@129.0.6668.58 /data/lib/browsers/chrome/mac_arm-129.0.6668.58/chrome-mac-arm64/Google Chrome for Testing.app/Contents/MacOS/Google Chrome for Testing
# playwright build v1010 downloaded to /home/squash/.cache/ms-playwright/ffmpeg-1010
output_lines = [
line for line in proc.stdout.strip().split('\n')
if '/chrom' in line
and 'chrom' in line.rsplit('/', 1)[-1].lower() # if final path segment (filename) contains chrome or chromium
and 'xdg-settings' not in line
and 'ffmpeg' not in line
]
if output_lines:
relpath = output_lines[0].split(str(self.playwright_browsers_dir))[-1]
abspath = self.playwright_browsers_dir / relpath
if os.path.isfile(abspath) and os.access(abspath, os.X_OK):
self._browser_abspaths[bin_name] = abspath
return (proc.stderr.strip() + "\n" + proc.stdout.strip()).strip()
PLAYWRIGHT_BINPROVIDER = PlaywrightBinProvider()

7
archivebox/vendor/abx-plugin-playwright/abx_plugin_playwright/config.py vendored Normal file
View File

@@ -0,0 +1,7 @@
from abx_spec_config import BaseConfigSet
class PlaywrightConfigs(BaseConfigSet):
PLAYWRIGHT_BINARY: str = 'playwright'
PLAYWRIGHT_CONFIG = PlaywrightConfigs()

20
archivebox/vendor/abx-plugin-playwright/pyproject.toml vendored Normal file
View File

@@ -0,0 +1,20 @@
[project]
name = "abx-plugin-playwright"
version = "2024.10.28"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"pydantic>=2.4.2",
"pydantic-pkgr>=0.5.4",
"abx-spec-pydantic-pkgr>=0.1.0",
"abx-spec-config>=0.1.0",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_playwright = "abx_plugin_playwright"

0
archivebox/vendor/abx-plugin-pocket/README.md vendored Normal file
View File

18
archivebox/vendor/abx-plugin-pocket/abx_plugin_pocket/__init__.py vendored Normal file
View File

@@ -0,0 +1,18 @@
__package__ = 'abx_plugin_pocket'
__label__ = 'Pocket'
import abx
@abx.hookimpl
def get_CONFIG():
from .config import POCKET_CONFIG
return {
'POCKET_CONFIG': POCKET_CONFIG
}
@abx.hookimpl
def ready():
from .config import POCKET_CONFIG
POCKET_CONFIG.validate()

15
archivebox/vendor/abx-plugin-pocket/abx_plugin_pocket/config.py vendored Normal file
View File

@@ -0,0 +1,15 @@
__package__ = 'abx_plugin_pocket'
from typing import Dict
from pydantic import Field
from abx_spec_config.base_configset import BaseConfigSet
class PocketConfig(BaseConfigSet):
POCKET_CONSUMER_KEY: str | None = Field(default=None)
POCKET_ACCESS_TOKENS: Dict[str, str] = Field(default=lambda: {}) # {<username>: <access_token>, ...}
POCKET_CONFIG = PocketConfig()

18
archivebox/vendor/abx-plugin-pocket/pyproject.toml vendored Normal file
View File

@@ -0,0 +1,18 @@
[project]
name = "abx-plugin-pocket"
version = "2024.10.28"
description = "Add your description here"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"abx>=0.1.0",
"abx-spec-config>=0.1.0",
"pocket>=0.3.6",
]
[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"
[project.entry-points.abx]
abx_plugin_pocket = "abx_plugin_pocket"

0
archivebox/vendor/abx-plugin-puppeteer/README.md vendored Normal file
View File

30
archivebox/vendor/abx-plugin-puppeteer/abx_plugin_puppeteer/__init__.py vendored Normal file
View File

@@ -0,0 +1,30 @@
__package__ = 'abx_plugin_puppeteer'
__label__ = 'Puppeteer'
__homepage__ = 'https://github.com/puppeteer/puppeteer'
import abx
@abx.hookimpl
def get_CONFIG():
from .config import PUPPETEER_CONFIG
return {
'PUPPETEER_CONFIG': PUPPETEER_CONFIG
}
@abx.hookimpl
def get_BINARIES():
from .binaries import PUPPETEER_BINARY
return {
'puppeteer': PUPPETEER_BINARY,
}
@abx.hookimpl
def get_BINPROVIDERS():
from .binproviders import PUPPETEER_BINPROVIDER
return {
'puppeteer': PUPPETEER_BINPROVIDER,
}

Some files were not shown because too many files have changed in this diff Show More