alex/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-04-06 07:47:53 +10:00
Code Issues Packages Projects Releases Wiki Activity

fix typing

Browse Source
This commit is contained in:
Nick Sweeting
2026-03-15 19:47:36 -07:00
parent 4756697a17
commit 44cabac8d0
7 changed files with 131 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
archivebox/config/common.py
View File

@@ -151,7 +151,7 @@ class ArchivingConfig(BaseConfigSet):
DEFAULT_PERSONA: str = Field(default="Default")
def validate(self):
def warn_if_invalid(self) -> None:
if int(self.TIMEOUT) < 5:
print(f"[red][!] Warning: TIMEOUT is set too low! (currently set to TIMEOUT={self.TIMEOUT} seconds)[/red]", file=sys.stderr)
print(" You must allow *at least* 5 seconds for indexing and archive methods to run succesfully.", file=sys.stderr)
@@ -165,10 +165,8 @@ class ArchivingConfig(BaseConfigSet):
def validate_check_ssl_validity(cls, v):
"""SIDE EFFECT: disable "you really shouldnt disable ssl" warnings emitted by requests"""
if not v:
import requests
import urllib3
requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
return v
@@ -206,6 +204,7 @@ class ArchivingConfig(BaseConfigSet):
ARCHIVING_CONFIG = ArchivingConfig()
ARCHIVING_CONFIG.warn_if_invalid()
class SearchBackendConfig(BaseConfigSet):

1
archivebox/config/django.py
View File

@@ -8,6 +8,7 @@ from datetime import datetime, timezone
from rich.console import Console
import django
import django.db
from archivebox.misc import logging

172
archivebox/config/views.py
View File

@@ -3,14 +3,15 @@ __package__ = 'archivebox.config'
import os
import shutil
import inspect
from typing import Any, List, Dict, cast
from typing import Any, List, Dict
from benedict import benedict
from django.http import HttpRequest
from django.utils import timezone
from django.utils.html import format_html, mark_safe
from django.utils.html import format_html
from django.utils.safestring import mark_safe
from admin_data_views.typing import TableContext, ItemContext
from admin_data_views.typing import TableContext, ItemContext, SectionData
from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink
from archivebox.config import CONSTANTS
@@ -29,6 +30,10 @@ KNOWN_BINARIES = [
]
def is_superuser(request: HttpRequest) -> bool:
return bool(getattr(request.user, 'is_superuser', False))
def obj_to_yaml(obj: Any, indent: int = 0) -> str:
indent_str = " " * indent
if indent == 0:
@@ -132,7 +137,7 @@ def get_filesystem_plugins() -> Dict[str, Dict[str, Any]]:
@render_with_table_view
def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
assert is_superuser(request), 'Must be a superuser to view configuration settings.'
rows = {
"Binary Name": [],
@@ -177,29 +182,27 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
@render_with_item_view
def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
assert request.user and request.user.is_superuser, 'Must be a superuser to view configuration settings.'
assert is_superuser(request), 'Must be a superuser to view configuration settings.'
# Try database first
try:
binary = Binary.objects.get(name=key)
section: SectionData = {
"name": binary.name,
"description": str(binary.abspath or ''),
"fields": {
'name': binary.name,
'binprovider': binary.binprovider,
'abspath': str(binary.abspath),
'version': binary.version,
'sha256': binary.sha256,
},
"help_texts": {},
}
return ItemContext(
slug=key,
title=key,
data=[
{
"name": binary.name,
"description": str(binary.abspath or ''),
"fields": {
'name': binary.name,
'binprovider': binary.binprovider,
'abspath': str(binary.abspath),
'version': binary.version,
'sha256': binary.sha256,
},
"help_texts": {},
},
],
data=[section],
)
except Binary.DoesNotExist:
pass
@@ -207,47 +210,44 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
# Try to detect from PATH
path = shutil.which(key)
if path:
section: SectionData = {
"name": key,
"description": path,
"fields": {
'name': key,
'binprovider': 'PATH',
'abspath': path,
'version': 'unknown',
},
"help_texts": {},
}
return ItemContext(
slug=key,
title=key,
data=[
{
"name": key,
"description": path,
"fields": {
'name': key,
'binprovider': 'PATH',
'abspath': path,
'version': 'unknown',
},
"help_texts": {},
},
],
data=[section],
)
section: SectionData = {
"name": key,
"description": "Binary not found",
"fields": {
'name': key,
'binprovider': 'not installed',
'abspath': 'not found',
'version': 'N/A',
},
"help_texts": {},
}
return ItemContext(
slug=key,
title=key,
data=[
{
"name": key,
"description": "Binary not found",
"fields": {
'name': key,
'binprovider': 'not installed',
'abspath': 'not found',
'version': 'N/A',
},
"help_texts": {},
},
],
data=[section],
)
@render_with_table_view
def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
assert is_superuser(request), 'Must be a superuser to view configuration settings.'
rows = {
"Name": [],
@@ -291,7 +291,7 @@ def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
import json
assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
assert is_superuser(request), 'Must be a superuser to view configuration settings.'
plugins = get_filesystem_plugins()
@@ -309,7 +309,7 @@ def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
"name": plugin['name'],
"source": plugin['source'],
"path": plugin['path'],
"hooks": plugin['hooks'],
"hooks": ', '.join(plugin['hooks']),
}
# Add config.json data if available
@@ -348,7 +348,7 @@ def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
@render_with_table_view
def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
assert request.user.is_superuser, "Must be a superuser to view configuration settings."
assert is_superuser(request), "Must be a superuser to view configuration settings."
rows = {
"Name": [],
@@ -369,8 +369,12 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
table=rows,
)
all_config_entries = cast(List[Dict[str, Any]], supervisor.getAllConfigInfo() or [])
all_config = {config["name"]: benedict(config) for config in all_config_entries}
all_config_entries = [
benedict(config)
for config in (supervisor.getAllConfigInfo() or [])
if isinstance(config, dict) and "name" in config
]
all_config = {str(config["name"]): config for config in all_config_entries}
# Add top row for supervisord process manager
rows["Name"].append(ItemLink('supervisord', key='supervisord'))
@@ -388,8 +392,10 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
rows['Exit Status'].append('0')
# Add a row for each worker process managed by supervisord
for proc in cast(List[Dict[str, Any]], supervisor.getAllProcessInfo()):
proc = benedict(proc)
for proc_data in supervisor.getAllProcessInfo():
if not isinstance(proc_data, dict):
continue
proc = benedict(proc_data)
rows["Name"].append(ItemLink(proc.name, key=proc.name))
rows["State"].append(proc.statename)
rows['PID'].append(proc.description.replace('pid ', ''))
@@ -412,7 +418,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
@render_with_item_view
def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
assert request.user.is_superuser, "Must be a superuser to view configuration settings."
assert is_superuser(request), "Must be a superuser to view configuration settings."
from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker, get_sock_file, CONFIG_FILE_NAME
@@ -427,11 +433,15 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
data=[],
)
all_config = cast(List[Dict[str, Any]], supervisor.getAllConfigInfo() or [])
all_config = [
benedict(config)
for config in (supervisor.getAllConfigInfo() or [])
if isinstance(config, dict)
]
if key == 'supervisord':
relevant_config = CONFIG_FILE.read_text()
relevant_logs = cast(str, supervisor.readLog(0, 10_000_000))
relevant_logs = str(supervisor.readLog(0, 10_000_000))
start_ts = [line for line in relevant_logs.split("\n") if "RPC interface 'supervisor' initialized" in line][-1].split(",", 1)[0]
uptime = str(timezone.now() - parse_date(start_ts)).split(".")[0]
@@ -449,37 +459,37 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
)
else:
proc = benedict(get_worker(supervisor, key) or {})
relevant_config = [config for config in all_config if config['name'] == key][0]
relevant_logs = supervisor.tailProcessStdoutLog(key, 0, 10_000_000)[0]
relevant_config = next((config for config in all_config if config.get('name') == key), benedict({}))
relevant_logs = str(supervisor.tailProcessStdoutLog(key, 0, 10_000_000)[0])
section: SectionData = {
"name": key,
"description": key,
"fields": {
"Command": str(proc.name),
"PID": str(proc.pid),
"State": str(proc.statename),
"Started": parse_date(proc.start).strftime("%Y-%m-%d %H:%M:%S") if proc.start else "",
"Stopped": parse_date(proc.stop).strftime("%Y-%m-%d %H:%M:%S") if proc.stop else "",
"Exit Status": str(proc.exitstatus),
"Logfile": str(proc.stdout_logfile),
"Uptime": str((proc.description or "").split("uptime ", 1)[-1]),
"Config": obj_to_yaml(dict(relevant_config)) if isinstance(relevant_config, dict) else str(relevant_config),
"Logs": relevant_logs,
},
"help_texts": {"Uptime": "How long the process has been running ([days:]hours:minutes:seconds)"},
}
return ItemContext(
slug=key,
title=key,
data=[
{
"name": key,
"description": key,
"fields": {
"Command": proc.name,
"PID": proc.pid,
"State": proc.statename,
"Started": parse_date(proc.start).strftime("%Y-%m-%d %H:%M:%S") if proc.start else "",
"Stopped": parse_date(proc.stop).strftime("%Y-%m-%d %H:%M:%S") if proc.stop else "",
"Exit Status": str(proc.exitstatus),
"Logfile": proc.stdout_logfile,
"Uptime": (proc.description or "").split("uptime ", 1)[-1],
"Config": relevant_config,
"Logs": relevant_logs,
},
"help_texts": {"Uptime": "How long the process has been running ([days:]hours:minutes:seconds)"},
},
],
data=[section],
)
@render_with_table_view
def log_list_view(request: HttpRequest, **kwargs) -> TableContext:
assert request.user.is_superuser, "Must be a superuser to view configuration settings."
assert is_superuser(request), "Must be a superuser to view configuration settings."
log_files = CONSTANTS.LOGS_DIR.glob("*.log")
@@ -516,7 +526,7 @@ def log_list_view(request: HttpRequest, **kwargs) -> TableContext:
@render_with_item_view
def log_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
assert request.user.is_superuser, "Must be a superuser to view configuration settings."
assert is_superuser(request), "Must be a superuser to view configuration settings."
log_file = [logfile for logfile in CONSTANTS.LOGS_DIR.glob('*.log') if key in logfile.name][0]

3
archivebox/core/admin_archiveresults.py
View File

@@ -4,7 +4,8 @@ import os
from pathlib import Path
from django.contrib import admin
from django.utils.html import format_html, mark_safe
from django.utils.html import format_html
from django.utils.safestring import mark_safe
from django.core.exceptions import ValidationError
from django.urls import reverse, resolve
from django.utils import timezone

28
archivebox/core/admin_snapshots.py
View File

@@ -6,7 +6,8 @@ from pathlib import Path
from django.contrib import admin, messages
from django.urls import path
from django.utils.html import format_html, mark_safe
from django.utils.html import format_html
from django.utils.safestring import mark_safe
from django.utils import timezone
from django.db.models import Q, Sum, Count, Prefetch
from django.db.models.functions import Coalesce
@@ -110,7 +111,9 @@ class SnapshotAdminForm(forms.ModelForm):
# Handle tags_editor field
if commit:
instance.save()
self._save_m2m()
save_m2m = getattr(self, '_save_m2m', None)
if callable(save_m2m):
save_m2m()
# Parse and save tags from tags_editor
tags_str = self.cleaned_data.get('tags_editor', '')
@@ -200,6 +203,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
def get_actions(self, request):
actions = super().get_actions(request)
if not actions:
return {}
if 'delete_selected' in actions:
func, name, _desc = actions['delete_selected']
actions['delete_selected'] = (func, name, 'Delete')
@@ -684,22 +689,23 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
# cl = self.get_changelist_instance(request)
# Save before monkey patching to restore for changelist list view
saved_change_list_template = self.change_list_template
saved_list_per_page = self.list_per_page
saved_list_max_show_all = self.list_max_show_all
admin_cls = type(self)
saved_change_list_template = admin_cls.change_list_template
saved_list_per_page = admin_cls.list_per_page
saved_list_max_show_all = admin_cls.list_max_show_all
# Monkey patch here plus core_tags.py
self.change_list_template = 'private_index_grid.html'
self.list_per_page = SERVER_CONFIG.SNAPSHOTS_PER_PAGE
self.list_max_show_all = self.list_per_page
admin_cls.change_list_template = 'private_index_grid.html'
admin_cls.list_per_page = SERVER_CONFIG.SNAPSHOTS_PER_PAGE
admin_cls.list_max_show_all = admin_cls.list_per_page
# Call monkey patched view
rendered_response = self.changelist_view(request, extra_context=extra_context)
# Restore values
self.change_list_template = saved_change_list_template
self.list_per_page = saved_list_per_page
self.list_max_show_all = saved_list_max_show_all
admin_cls.change_list_template = saved_change_list_template
admin_cls.list_per_page = saved_list_per_page
admin_cls.list_max_show_all = saved_list_max_show_all
return rendered_response

15
archivebox/tests/test_cli_shell.py
View File

@@ -22,3 +22,18 @@ def test_shell_command_exists(tmp_path, process):
# Should show shell help or recognize command
assert result.returncode in [0, 1, 2]
def test_shell_c_executes_python(tmp_path, process):
"""shell -c should fully initialize Django and run the provided command."""
os.chdir(tmp_path)
result = subprocess.run(
['archivebox', 'shell', '-c', 'print("shell-ok")'],
capture_output=True,
text=True,
timeout=30,
)
assert result.returncode == 0, result.stderr
assert 'shell-ok' in result.stdout

5
archivebox/workers/models.py
View File

@@ -26,8 +26,6 @@ ObjectStateList = Iterable[ObjectState]
class BaseModelWithStateMachine(models.Model, MachineMixin):
id: models.UUIDField
StatusChoices: ClassVar[Type[DefaultStatusChoices]]
# status: models.CharField
@@ -384,6 +382,9 @@ class ModelWithStateMachine(BaseModelWithStateMachine):
active_state = StatusChoices.STARTED
retry_at_field_name: str = 'retry_at'
class Meta(BaseModelWithStateMachine.Meta):
abstract = True
class BaseStateMachine(StateMachine):
"""
Base class for all ArchiveBox state machines.