fix typing

archivebox/config/common.py

@@ -151,7 +151,7 @@ class ArchivingConfig(BaseConfigSet):
 
     DEFAULT_PERSONA: str = Field(default="Default")
 
-    def validate(self):
+    def warn_if_invalid(self) -> None:
         if int(self.TIMEOUT) < 5:
             print(f"[red][!] Warning: TIMEOUT is set too low! (currently set to TIMEOUT={self.TIMEOUT} seconds)[/red]", file=sys.stderr)
             print("    You must allow *at least* 5 seconds for indexing and archive methods to run succesfully.", file=sys.stderr)
@@ -165,10 +165,8 @@ class ArchivingConfig(BaseConfigSet):
     def validate_check_ssl_validity(cls, v):
         """SIDE EFFECT: disable "you really shouldnt disable ssl" warnings emitted by requests"""
         if not v:
-            import requests
             import urllib3
 
-            requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
             urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
         return v
 
@@ -206,6 +204,7 @@ class ArchivingConfig(BaseConfigSet):
 
 
 ARCHIVING_CONFIG = ArchivingConfig()
+ARCHIVING_CONFIG.warn_if_invalid()
 
 
 class SearchBackendConfig(BaseConfigSet):

archivebox/config/django.py

@@ -8,6 +8,7 @@ from datetime import datetime, timezone
 from rich.console import Console
 
 import django
+import django.db
 
 from archivebox.misc import logging
 

archivebox/config/views.py

@@ -3,14 +3,15 @@ __package__ = 'archivebox.config'
 import os
 import shutil
 import inspect
-from typing import Any, List, Dict, cast
+from typing import Any, List, Dict
 from benedict import benedict
 
 from django.http import HttpRequest
 from django.utils import timezone
-from django.utils.html import format_html, mark_safe
+from django.utils.html import format_html
+from django.utils.safestring import mark_safe
 
-from admin_data_views.typing import TableContext, ItemContext
+from admin_data_views.typing import TableContext, ItemContext, SectionData
 from admin_data_views.utils import render_with_table_view, render_with_item_view, ItemLink
 
 from archivebox.config import CONSTANTS
@@ -29,6 +30,10 @@ KNOWN_BINARIES = [
 ]
 
 
+def is_superuser(request: HttpRequest) -> bool:
+    return bool(getattr(request.user, 'is_superuser', False))
+
+
 def obj_to_yaml(obj: Any, indent: int = 0) -> str:
     indent_str = "  " * indent
     if indent == 0:
@@ -132,7 +137,7 @@ def get_filesystem_plugins() -> Dict[str, Dict[str, Any]]:
 
 @render_with_table_view
 def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
-    assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
+    assert is_superuser(request), 'Must be a superuser to view configuration settings.'
 
     rows = {
         "Binary Name": [],
@@ -177,29 +182,27 @@ def binaries_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
 @render_with_item_view
 def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
-
+    assert is_superuser(request), 'Must be a superuser to view configuration settings.'
-    assert request.user and request.user.is_superuser, 'Must be a superuser to view configuration settings.'
 
     # Try database first
     try:
         binary = Binary.objects.get(name=key)
+        section: SectionData = {
+            "name": binary.name,
+            "description": str(binary.abspath or ''),
+            "fields": {
+                'name': binary.name,
+                'binprovider': binary.binprovider,
+                'abspath': str(binary.abspath),
+                'version': binary.version,
+                'sha256': binary.sha256,
+            },
+            "help_texts": {},
+        }
         return ItemContext(
             slug=key,
             title=key,
-            data=[
+            data=[section],
-                {
-                    "name": binary.name,
-                    "description": str(binary.abspath or ''),
-                    "fields": {
-                        'name': binary.name,
-                        'binprovider': binary.binprovider,
-                        'abspath': str(binary.abspath),
-                        'version': binary.version,
-                        'sha256': binary.sha256,
-                    },
-                    "help_texts": {},
-                },
-            ],
         )
     except Binary.DoesNotExist:
         pass
@@ -207,47 +210,44 @@ def binary_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
     # Try to detect from PATH
     path = shutil.which(key)
     if path:
+        section: SectionData = {
+            "name": key,
+            "description": path,
+            "fields": {
+                'name': key,
+                'binprovider': 'PATH',
+                'abspath': path,
+                'version': 'unknown',
+            },
+            "help_texts": {},
+        }
         return ItemContext(
             slug=key,
             title=key,
-            data=[
+            data=[section],
-                {
-                    "name": key,
-                    "description": path,
-                    "fields": {
-                        'name': key,
-                        'binprovider': 'PATH',
-                        'abspath': path,
-                        'version': 'unknown',
-                    },
-                    "help_texts": {},
-                },
-            ],
         )
 
+    section: SectionData = {
+        "name": key,
+        "description": "Binary not found",
+        "fields": {
+            'name': key,
+            'binprovider': 'not installed',
+            'abspath': 'not found',
+            'version': 'N/A',
+        },
+        "help_texts": {},
+    }
     return ItemContext(
         slug=key,
         title=key,
-        data=[
+        data=[section],
-            {
-                "name": key,
-                "description": "Binary not found",
-                "fields": {
-                    'name': key,
-                    'binprovider': 'not installed',
-                    'abspath': 'not found',
-                    'version': 'N/A',
-                },
-                "help_texts": {},
-            },
-        ],
     )
 
 
 @render_with_table_view
 def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
-
+    assert is_superuser(request), 'Must be a superuser to view configuration settings.'
-    assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
 
     rows = {
         "Name": [],
@@ -291,7 +291,7 @@ def plugins_list_view(request: HttpRequest, **kwargs) -> TableContext:
 def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
     import json
 
-    assert request.user.is_superuser, 'Must be a superuser to view configuration settings.'
+    assert is_superuser(request), 'Must be a superuser to view configuration settings.'
 
     plugins = get_filesystem_plugins()
 
@@ -309,7 +309,7 @@ def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
         "name": plugin['name'],
         "source": plugin['source'],
         "path": plugin['path'],
-        "hooks": plugin['hooks'],
+        "hooks": ', '.join(plugin['hooks']),
     }
 
     # Add config.json data if available
@@ -348,7 +348,7 @@ def plugin_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
 
 @render_with_table_view
 def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
-    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
+    assert is_superuser(request), "Must be a superuser to view configuration settings."
 
     rows = {
         "Name": [],
@@ -369,8 +369,12 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
             table=rows,
         )
 
-    all_config_entries = cast(List[Dict[str, Any]], supervisor.getAllConfigInfo() or [])
+    all_config_entries = [
-    all_config = {config["name"]: benedict(config) for config in all_config_entries}
+        benedict(config)
+        for config in (supervisor.getAllConfigInfo() or [])
+        if isinstance(config, dict) and "name" in config
+    ]
+    all_config = {str(config["name"]): config for config in all_config_entries}
 
     # Add top row for supervisord process manager
     rows["Name"].append(ItemLink('supervisord', key='supervisord'))
@@ -388,8 +392,10 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
     rows['Exit Status'].append('0')
 
     # Add a row for each worker process managed by supervisord
-    for proc in cast(List[Dict[str, Any]], supervisor.getAllProcessInfo()):
+    for proc_data in supervisor.getAllProcessInfo():
-        proc = benedict(proc)
+        if not isinstance(proc_data, dict):
+            continue
+        proc = benedict(proc_data)
         rows["Name"].append(ItemLink(proc.name, key=proc.name))
         rows["State"].append(proc.statename)
         rows['PID'].append(proc.description.replace('pid ', ''))
@@ -412,7 +418,7 @@ def worker_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
 @render_with_item_view
 def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
-    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
+    assert is_superuser(request), "Must be a superuser to view configuration settings."
 
     from archivebox.workers.supervisord_util import get_existing_supervisord_process, get_worker, get_sock_file, CONFIG_FILE_NAME
 
@@ -427,11 +433,15 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
             data=[],
         )
 
-    all_config = cast(List[Dict[str, Any]], supervisor.getAllConfigInfo() or [])
+    all_config = [
+        benedict(config)
+        for config in (supervisor.getAllConfigInfo() or [])
+        if isinstance(config, dict)
+    ]
 
     if key == 'supervisord':
         relevant_config = CONFIG_FILE.read_text()
-        relevant_logs = cast(str, supervisor.readLog(0, 10_000_000))
+        relevant_logs = str(supervisor.readLog(0, 10_000_000))
         start_ts = [line for line in relevant_logs.split("\n") if "RPC interface 'supervisor' initialized" in line][-1].split(",", 1)[0]
         uptime = str(timezone.now() - parse_date(start_ts)).split(".")[0]
 
@@ -449,37 +459,37 @@ def worker_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
         )
     else:
         proc = benedict(get_worker(supervisor, key) or {})
-        relevant_config = [config for config in all_config if config['name'] == key][0]
+        relevant_config = next((config for config in all_config if config.get('name') == key), benedict({}))
-        relevant_logs = supervisor.tailProcessStdoutLog(key, 0, 10_000_000)[0]
+        relevant_logs = str(supervisor.tailProcessStdoutLog(key, 0, 10_000_000)[0])
+
+    section: SectionData = {
+        "name": key,
+        "description": key,
+        "fields": {
+            "Command": str(proc.name),
+            "PID": str(proc.pid),
+            "State": str(proc.statename),
+            "Started": parse_date(proc.start).strftime("%Y-%m-%d %H:%M:%S") if proc.start else "",
+            "Stopped": parse_date(proc.stop).strftime("%Y-%m-%d %H:%M:%S") if proc.stop else "",
+            "Exit Status": str(proc.exitstatus),
+            "Logfile": str(proc.stdout_logfile),
+            "Uptime": str((proc.description or "").split("uptime ", 1)[-1]),
+            "Config": obj_to_yaml(dict(relevant_config)) if isinstance(relevant_config, dict) else str(relevant_config),
+            "Logs": relevant_logs,
+        },
+        "help_texts": {"Uptime": "How long the process has been running ([days:]hours:minutes:seconds)"},
+    }
 
     return ItemContext(
         slug=key,
         title=key,
-        data=[
+        data=[section],
-            {
-                "name": key,
-                "description": key,
-                "fields": {
-                    "Command": proc.name,
-                    "PID": proc.pid,
-                    "State": proc.statename,
-                    "Started": parse_date(proc.start).strftime("%Y-%m-%d %H:%M:%S") if proc.start else "",
-                    "Stopped": parse_date(proc.stop).strftime("%Y-%m-%d %H:%M:%S") if proc.stop else "",
-                    "Exit Status": str(proc.exitstatus),
-                    "Logfile": proc.stdout_logfile,
-                    "Uptime": (proc.description or "").split("uptime ", 1)[-1],
-                    "Config": relevant_config,
-                    "Logs": relevant_logs,
-                },
-                "help_texts": {"Uptime": "How long the process has been running ([days:]hours:minutes:seconds)"},
-            },
-        ],
     )
 
 
 @render_with_table_view
 def log_list_view(request: HttpRequest, **kwargs) -> TableContext:
-    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
+    assert is_superuser(request), "Must be a superuser to view configuration settings."
 
 
     log_files = CONSTANTS.LOGS_DIR.glob("*.log")
@@ -516,7 +526,7 @@ def log_list_view(request: HttpRequest, **kwargs) -> TableContext:
 
 @render_with_item_view
 def log_detail_view(request: HttpRequest, key: str, **kwargs) -> ItemContext:
-    assert request.user.is_superuser, "Must be a superuser to view configuration settings."
+    assert is_superuser(request), "Must be a superuser to view configuration settings."
 
     log_file = [logfile for logfile in CONSTANTS.LOGS_DIR.glob('*.log') if key in logfile.name][0]
 

archivebox/core/admin_archiveresults.py

@@ -4,7 +4,8 @@ import os
 from pathlib import Path
 
 from django.contrib import admin
-from django.utils.html import format_html, mark_safe
+from django.utils.html import format_html
+from django.utils.safestring import mark_safe
 from django.core.exceptions import ValidationError
 from django.urls import reverse, resolve
 from django.utils import timezone

archivebox/core/admin_snapshots.py

@@ -6,7 +6,8 @@ from pathlib import Path
 
 from django.contrib import admin, messages
 from django.urls import path
-from django.utils.html import format_html, mark_safe
+from django.utils.html import format_html
+from django.utils.safestring import mark_safe
 from django.utils import timezone
 from django.db.models import Q, Sum, Count, Prefetch
 from django.db.models.functions import Coalesce
@@ -110,7 +111,9 @@ class SnapshotAdminForm(forms.ModelForm):
         # Handle tags_editor field
         if commit:
             instance.save()
-            self._save_m2m()
+            save_m2m = getattr(self, '_save_m2m', None)
+            if callable(save_m2m):
+                save_m2m()
 
             # Parse and save tags from tags_editor
             tags_str = self.cleaned_data.get('tags_editor', '')
@@ -200,6 +203,8 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
 
     def get_actions(self, request):
         actions = super().get_actions(request)
+        if not actions:
+            return {}
         if 'delete_selected' in actions:
             func, name, _desc = actions['delete_selected']
             actions['delete_selected'] = (func, name, 'Delete')
@@ -684,22 +689,23 @@ class SnapshotAdmin(SearchResultsAdminMixin, ConfigEditorMixin, BaseModelAdmin):
         # cl = self.get_changelist_instance(request)
 
         # Save before monkey patching to restore for changelist list view
-        saved_change_list_template = self.change_list_template
+        admin_cls = type(self)
-        saved_list_per_page = self.list_per_page
+        saved_change_list_template = admin_cls.change_list_template
-        saved_list_max_show_all = self.list_max_show_all
+        saved_list_per_page = admin_cls.list_per_page
+        saved_list_max_show_all = admin_cls.list_max_show_all
 
         # Monkey patch here plus core_tags.py
-        self.change_list_template = 'private_index_grid.html'
+        admin_cls.change_list_template = 'private_index_grid.html'
-        self.list_per_page = SERVER_CONFIG.SNAPSHOTS_PER_PAGE
+        admin_cls.list_per_page = SERVER_CONFIG.SNAPSHOTS_PER_PAGE
-        self.list_max_show_all = self.list_per_page
+        admin_cls.list_max_show_all = admin_cls.list_per_page
 
         # Call monkey patched view
         rendered_response = self.changelist_view(request, extra_context=extra_context)
 
         # Restore values
-        self.change_list_template = saved_change_list_template
+        admin_cls.change_list_template = saved_change_list_template
-        self.list_per_page = saved_list_per_page
+        admin_cls.list_per_page = saved_list_per_page
-        self.list_max_show_all = saved_list_max_show_all
+        admin_cls.list_max_show_all = saved_list_max_show_all
 
         return rendered_response
 

archivebox/tests/test_cli_shell.py

@@ -22,3 +22,18 @@ def test_shell_command_exists(tmp_path, process):
 
     # Should show shell help or recognize command
     assert result.returncode in [0, 1, 2]
+
+
+def test_shell_c_executes_python(tmp_path, process):
+    """shell -c should fully initialize Django and run the provided command."""
+    os.chdir(tmp_path)
+
+    result = subprocess.run(
+        ['archivebox', 'shell', '-c', 'print("shell-ok")'],
+        capture_output=True,
+        text=True,
+        timeout=30,
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert 'shell-ok' in result.stdout

archivebox/workers/models.py

@@ -26,8 +26,6 @@ ObjectStateList = Iterable[ObjectState]
 
 
 class BaseModelWithStateMachine(models.Model, MachineMixin):
-    id: models.UUIDField
-
     StatusChoices: ClassVar[Type[DefaultStatusChoices]]
 
     # status: models.CharField
@@ -384,6 +382,9 @@ class ModelWithStateMachine(BaseModelWithStateMachine):
     active_state = StatusChoices.STARTED
     retry_at_field_name: str               = 'retry_at'
 
+    class Meta(BaseModelWithStateMachine.Meta):
+        abstract = True
+
 class BaseStateMachine(StateMachine):
     """
     Base class for all ArchiveBox state machines.