wip

archivebox/workers/__init__.py

@@ -1,7 +1,8 @@
-__package__ = 'archivebox.workers'
+__package__ = "archivebox.workers"
 __order__ = 100
 
 
 def register_admin(admin_site):
     from archivebox.workers.admin import register_admin
+
     register_admin(admin_site)

archivebox/workers/admin.py

@@ -4,7 +4,7 @@ Workers admin module.
 Background runner processes do not need Django admin registration.
 """
 
-__package__ = 'archivebox.workers'
+__package__ = "archivebox.workers"
 
 
 def register_admin(admin_site):

archivebox/workers/apps.py

@@ -2,7 +2,6 @@ from django.apps import AppConfig
 
 
 class WorkersConfig(AppConfig):
-    default_auto_field = 'django.db.models.BigAutoField'
-    name = 'archivebox.workers'
-    label = 'workers'
-
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "archivebox.workers"
+    label = "workers"

archivebox/workers/management/commands/runner_watch.py

@@ -72,6 +72,7 @@ class Command(BaseCommand):
 
         def restart_runner() -> None:
             Process.cleanup_stale_running()
+            Process.cleanup_orphaned_workers()
             machine = Machine.current()
 
             running = Process.objects.filter(
@@ -105,7 +106,7 @@ class Command(BaseCommand):
         while True:
             try:
                 if os.path.exists(pidfile):
-                    with open(pidfile, "r") as handle:
+                    with open(pidfile) as handle:
                         pid = handle.read().strip() or None
                 else:
                     pid = None

archivebox/workers/models.py

@@ -1,6 +1,7 @@
-__package__ = 'archivebox.workers'
+__package__ = "archivebox.workers"
 
-from typing import ClassVar, Type, Iterable
+from typing import ClassVar
+from collections.abc import Iterable
 from datetime import datetime, timedelta
 from statemachine.mixins import MachineMixin
 
@@ -14,12 +15,19 @@ from statemachine import registry, StateMachine, State
 
 
 class DefaultStatusChoices(models.TextChoices):
-    QUEUED = 'queued', 'Queued'
-    STARTED = 'started', 'Started'
-    SEALED = 'sealed', 'Sealed'
+    QUEUED = "queued", "Queued"
+    STARTED = "started", "Started"
+    SEALED = "sealed", "Sealed"
 
 
-default_status_field: models.CharField = models.CharField(choices=DefaultStatusChoices.choices, max_length=15, default=DefaultStatusChoices.QUEUED, null=False, blank=False, db_index=True)
+default_status_field: models.CharField = models.CharField(
+    choices=DefaultStatusChoices.choices,
+    max_length=15,
+    default=DefaultStatusChoices.QUEUED,
+    null=False,
+    blank=False,
+    db_index=True,
+)
 default_retry_at_field: models.DateTimeField = models.DateTimeField(default=timezone.now, null=True, blank=True, db_index=True)
 
 ObjectState = State | str
@@ -27,21 +35,21 @@ ObjectStateList = Iterable[ObjectState]
 
 
 class BaseModelWithStateMachine(models.Model, MachineMixin):
-    StatusChoices: ClassVar[Type[DefaultStatusChoices]]
+    StatusChoices: ClassVar[type[DefaultStatusChoices]]
 
     # status: models.CharField
     # retry_at: models.DateTimeField
 
     state_machine_name: str | None = None
     state_field_name: str
-    state_machine_attr: str = 'sm'
+    state_machine_attr: str = "sm"
     bind_events_as_methods: bool = True
 
     active_state: ObjectState
     retry_at_field_name: str
 
     class Meta(TypedModelMeta):
-        app_label = 'workers'
+        app_label = "workers"
         abstract = True
 
     @classmethod
@@ -49,7 +57,7 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
         import sys
 
         # Skip state machine checks during makemigrations to avoid premature registry access
-        if 'makemigrations' in sys.argv:
+        if "makemigrations" in sys.argv:
             return super().check(**kwargs)
 
         errors = super().check(**kwargs)
@@ -59,88 +67,105 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
         found_retry_at_field = False
 
         for field in cls._meta.get_fields():
-            if getattr(field, '_is_state_field', False):
+            if getattr(field, "_is_state_field", False):
                 if cls.state_field_name == field.name:
                     found_status_field = True
-                    if getattr(field, 'choices', None) != cls.StatusChoices.choices:
-                        errors.append(checks.Error(
-                            f'{cls.__name__}.{field.name} must have choices set to {cls.__name__}.StatusChoices.choices',
-                            hint=f'{cls.__name__}.{field.name}.choices = {getattr(field, "choices", None)!r}',
-                            obj=cls,
-                            id='workers.E011',
-                        ))
-            if getattr(field, '_is_retry_at_field', False):
+                    if getattr(field, "choices", None) != cls.StatusChoices.choices:
+                        errors.append(
+                            checks.Error(
+                                f"{cls.__name__}.{field.name} must have choices set to {cls.__name__}.StatusChoices.choices",
+                                hint=f"{cls.__name__}.{field.name}.choices = {getattr(field, 'choices', None)!r}",
+                                obj=cls,
+                                id="workers.E011",
+                            ),
+                        )
+            if getattr(field, "_is_retry_at_field", False):
                 if cls.retry_at_field_name == field.name:
                     found_retry_at_field = True
-            if field.name == 'id' and getattr(field, 'primary_key', False):
+            if field.name == "id" and getattr(field, "primary_key", False):
                 found_id_field = True
 
         if not found_status_field:
-            errors.append(checks.Error(
-                f'{cls.__name__}.state_field_name must be defined and point to a StatusField()',
-                hint=f'{cls.__name__}.state_field_name = {cls.state_field_name!r} but {cls.__name__}.{cls.state_field_name!r} was not found or does not refer to StatusField',
-                obj=cls,
-                id='workers.E012',
-            ))
+            errors.append(
+                checks.Error(
+                    f"{cls.__name__}.state_field_name must be defined and point to a StatusField()",
+                    hint=f"{cls.__name__}.state_field_name = {cls.state_field_name!r} but {cls.__name__}.{cls.state_field_name!r} was not found or does not refer to StatusField",
+                    obj=cls,
+                    id="workers.E012",
+                ),
+            )
         if not found_retry_at_field:
-            errors.append(checks.Error(
-                f'{cls.__name__}.retry_at_field_name must be defined and point to a RetryAtField()',
-                hint=f'{cls.__name__}.retry_at_field_name = {cls.retry_at_field_name!r} but {cls.__name__}.{cls.retry_at_field_name!r} was not found or does not refer to RetryAtField',
-                obj=cls,
-                id='workers.E013',
-            ))
+            errors.append(
+                checks.Error(
+                    f"{cls.__name__}.retry_at_field_name must be defined and point to a RetryAtField()",
+                    hint=f"{cls.__name__}.retry_at_field_name = {cls.retry_at_field_name!r} but {cls.__name__}.{cls.retry_at_field_name!r} was not found or does not refer to RetryAtField",
+                    obj=cls,
+                    id="workers.E013",
+                ),
+            )
 
         if not found_id_field:
-            errors.append(checks.Error(
-                f'{cls.__name__} must have an id field that is a primary key',
-                hint=f'{cls.__name__}.id field missing or not configured as primary key',
-                obj=cls,
-                id='workers.E014',
-            ))
+            errors.append(
+                checks.Error(
+                    f"{cls.__name__} must have an id field that is a primary key",
+                    hint=f"{cls.__name__}.id field missing or not configured as primary key",
+                    obj=cls,
+                    id="workers.E014",
+                ),
+            )
 
         if not isinstance(cls.state_machine_name, str):
-            errors.append(checks.Error(
-                f'{cls.__name__}.state_machine_name must be a dotted-import path to a StateMachine class',
-                hint=f'{cls.__name__}.state_machine_name = {cls.state_machine_name!r}',
-                obj=cls,
-                id='workers.E015',
-            ))
+            errors.append(
+                checks.Error(
+                    f"{cls.__name__}.state_machine_name must be a dotted-import path to a StateMachine class",
+                    hint=f"{cls.__name__}.state_machine_name = {cls.state_machine_name!r}",
+                    obj=cls,
+                    id="workers.E015",
+                ),
+            )
 
         try:
             cls.StateMachineClass
         except Exception as err:
-            errors.append(checks.Error(
-                f'{cls.__name__}.state_machine_name must point to a valid StateMachine class, but got {type(err).__name__} {err} when trying to access {cls.__name__}.StateMachineClass',
-                hint=f'{cls.__name__}.state_machine_name = {cls.state_machine_name!r}',
-                obj=cls,
-                id='workers.E016',
-            ))
+            errors.append(
+                checks.Error(
+                    f"{cls.__name__}.state_machine_name must point to a valid StateMachine class, but got {type(err).__name__} {err} when trying to access {cls.__name__}.StateMachineClass",
+                    hint=f"{cls.__name__}.state_machine_name = {cls.state_machine_name!r}",
+                    obj=cls,
+                    id="workers.E016",
+                ),
+            )
 
         if cls.INITIAL_STATE not in cls.StatusChoices.values:
-            errors.append(checks.Error(
-                f'{cls.__name__}.StateMachineClass.initial_state must be present within {cls.__name__}.StatusChoices',
-                hint=f'{cls.__name__}.StateMachineClass.initial_state = {cls.StateMachineClass.initial_state!r}',
-                obj=cls,
-                id='workers.E017',
-            ))
+            errors.append(
+                checks.Error(
+                    f"{cls.__name__}.StateMachineClass.initial_state must be present within {cls.__name__}.StatusChoices",
+                    hint=f"{cls.__name__}.StateMachineClass.initial_state = {cls.StateMachineClass.initial_state!r}",
+                    obj=cls,
+                    id="workers.E017",
+                ),
+            )
 
         if cls.ACTIVE_STATE not in cls.StatusChoices.values:
-            errors.append(checks.Error(
-                f'{cls.__name__}.active_state must be set to a valid State present within {cls.__name__}.StatusChoices',
-                hint=f'{cls.__name__}.active_state = {cls.active_state!r}',
-                obj=cls,
-                id='workers.E018',
-            ))
-
+            errors.append(
+                checks.Error(
+                    f"{cls.__name__}.active_state must be set to a valid State present within {cls.__name__}.StatusChoices",
+                    hint=f"{cls.__name__}.active_state = {cls.active_state!r}",
+                    obj=cls,
+                    id="workers.E018",
+                ),
+            )
 
         for state in cls.FINAL_STATES:
             if state not in cls.StatusChoices.values:
-                errors.append(checks.Error(
-                    f'{cls.__name__}.StateMachineClass.final_states must all be present within {cls.__name__}.StatusChoices',
-                    hint=f'{cls.__name__}.StateMachineClass.final_states = {cls.StateMachineClass.final_states!r}',
-                    obj=cls,
-                    id='workers.E019',
-                ))
+                errors.append(
+                    checks.Error(
+                        f"{cls.__name__}.StateMachineClass.final_states must all be present within {cls.__name__}.StatusChoices",
+                        hint=f"{cls.__name__}.StateMachineClass.final_states = {cls.StateMachineClass.final_states!r}",
+                        obj=cls,
+                        id="workers.E019",
+                    ),
+                )
                 break
         return errors
 
@@ -149,7 +174,6 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
         """Convert a statemachine.State, models.TextChoices.choices value, or Enum value to a str"""
         return str(state.value) if isinstance(state, State) else str(state)
 
-
     @property
     def RETRY_AT(self) -> datetime:
         return getattr(self, self.retry_at_field_name)
@@ -182,10 +206,14 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
             setattr(self, key, value)
 
         # Try to save with optimistic locking
-        updated = type(self).objects.filter(
-            pk=self.pk,
-            retry_at=current_retry_at,
-        ).update(**{k: getattr(self, k) for k in kwargs})
+        updated = (
+            type(self)
+            .objects.filter(
+                pk=self.pk,
+                retry_at=current_retry_at,
+            )
+            .update(**{k: getattr(self, k) for k in kwargs})
+        )
 
         if updated == 1:
             self.refresh_from_db()
@@ -200,14 +228,18 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
         - status is not in FINAL_STATES
         - retry_at is in the past (or now)
         """
-        return cls.objects.filter(
-            retry_at__lte=timezone.now()
-        ).exclude(
-            status__in=cls.FINAL_STATES
-        ).order_by('retry_at')
+        return (
+            cls.objects.filter(
+                retry_at__lte=timezone.now(),
+            )
+            .exclude(
+                status__in=cls.FINAL_STATES,
+            )
+            .order_by("retry_at")
+        )
 
     @classmethod
-    def claim_for_worker(cls, obj: 'BaseModelWithStateMachine', lock_seconds: int = 60) -> bool:
+    def claim_for_worker(cls, obj: "BaseModelWithStateMachine", lock_seconds: int = 60) -> bool:
         """
         Atomically claim a due object for processing using retry_at as the lock.
 
@@ -231,7 +263,7 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
             retry_at=obj.RETRY_AT,
             retry_at__lte=timezone.now(),
         ).update(
-            retry_at=timezone.now() + timedelta(seconds=lock_seconds)
+            retry_at=timezone.now() + timedelta(seconds=lock_seconds),
         )
         return updated == 1
 
@@ -270,9 +302,9 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
         if not self.claim_processing_lock(lock_seconds=lock_seconds):
             return False
 
-        tick = getattr(getattr(self, self.state_machine_attr, None), 'tick', None)
+        tick = getattr(getattr(self, self.state_machine_attr, None), "tick", None)
         if not callable(tick):
-            raise TypeError(f'{type(self).__name__}.{self.state_machine_attr}.tick() must be callable')
+            raise TypeError(f"{type(self).__name__}.{self.state_machine_attr}.tick() must be callable")
         tick()
         self.refresh_from_db()
         return True
@@ -285,7 +317,7 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
     def INITIAL_STATE(cls) -> str:
         initial_state = cls.StateMachineClass.initial_state
         if initial_state is None:
-            raise ValueError('StateMachineClass.initial_state must not be None')
+            raise ValueError("StateMachineClass.initial_state must not be None")
         return cls._state_to_str(initial_state)
 
     @classproperty
@@ -297,7 +329,7 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
         return [*cls.FINAL_STATES, cls.ACTIVE_STATE]
 
     @classmethod
-    def extend_choices(cls, base_choices: Type[models.TextChoices]):
+    def extend_choices(cls, base_choices: type[models.TextChoices]):
         """
         Decorator to extend the base choices with extra choices, e.g.:
 
@@ -309,16 +341,20 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
                 FAILED = 'failed'
                 SKIPPED = 'skipped'
         """
-        assert issubclass(base_choices, models.TextChoices), f'@extend_choices(base_choices) must be a TextChoices class, not {base_choices.__name__}'
-        def wrapper(extra_choices: Type[models.TextChoices]) -> Type[models.TextChoices]:
+        assert issubclass(base_choices, models.TextChoices), (
+            f"@extend_choices(base_choices) must be a TextChoices class, not {base_choices.__name__}"
+        )
+
+        def wrapper(extra_choices: type[models.TextChoices]) -> type[models.TextChoices]:
             joined = {}
             for item in base_choices.choices:
                 joined[item[0]] = item[1]
             for item in extra_choices.choices:
                 joined[item[0]] = item[1]
-            joined_choices = models.TextChoices('StatusChoices', joined)
+            joined_choices = models.TextChoices("StatusChoices", joined)
             assert isinstance(joined_choices, type)
             return joined_choices
+
         return wrapper
 
     @classmethod
@@ -340,7 +376,7 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
         default_kwargs = default_status_field.deconstruct()[3]
         updated_kwargs = {**default_kwargs, **kwargs}
         field = models.CharField(**updated_kwargs)
-        field._is_state_field = True                    # type: ignore
+        field._is_state_field = True  # type: ignore
         return field
 
     @classmethod
@@ -354,19 +390,19 @@ class BaseModelWithStateMachine(models.Model, MachineMixin):
         default_kwargs = default_retry_at_field.deconstruct()[3]
         updated_kwargs = {**default_kwargs, **kwargs}
         field = models.DateTimeField(**updated_kwargs)
-        field._is_retry_at_field = True                 # type: ignore
+        field._is_retry_at_field = True  # type: ignore
         return field
 
     @classproperty
-    def StateMachineClass(cls) -> Type[StateMachine]:
+    def StateMachineClass(cls) -> type[StateMachine]:
         """Get the StateMachine class for the given django Model that inherits from MachineMixin"""
 
-        model_state_machine_name = getattr(cls, 'state_machine_name', None)
+        model_state_machine_name = getattr(cls, "state_machine_name", None)
         if model_state_machine_name:
             StateMachineCls = registry.get_machine_cls(model_state_machine_name)
             assert issubclass(StateMachineCls, StateMachine)
             return StateMachineCls
-        raise NotImplementedError('ActorType must define .state_machine_name that points to a valid StateMachine')
+        raise NotImplementedError("ActorType must define .state_machine_name that points to a valid StateMachine")
 
 
 class ModelWithStateMachine(BaseModelWithStateMachine):
@@ -375,17 +411,18 @@ class ModelWithStateMachine(BaseModelWithStateMachine):
     status: models.CharField = BaseModelWithStateMachine.StatusField()
     retry_at: models.DateTimeField = BaseModelWithStateMachine.RetryAtField()
 
-    state_machine_name: str | None         # e.g. 'core.models.ArchiveResultMachine'
-    state_field_name: str                  = 'status'
-    state_machine_attr: str                = 'sm'
-    bind_events_as_methods: bool           = True
+    state_machine_name: str | None  # e.g. 'core.models.ArchiveResultMachine'
+    state_field_name: str = "status"
+    state_machine_attr: str = "sm"
+    bind_events_as_methods: bool = True
 
     active_state = StatusChoices.STARTED
-    retry_at_field_name: str               = 'retry_at'
+    retry_at_field_name: str = "retry_at"
 
     class Meta(BaseModelWithStateMachine.Meta):
         abstract = True
 
+
 class BaseStateMachine(StateMachine):
     """
     Base class for all ArchiveBox state machines.
@@ -408,7 +445,7 @@ class BaseStateMachine(StateMachine):
     (e.g., self.snapshot, self.archiveresult, etc.)
     """
 
-    model_attr_name: str = 'obj'  # Override in subclasses
+    model_attr_name: str = "obj"  # Override in subclasses
 
     def __init__(self, obj, *args, **kwargs):
         setattr(self, self.model_attr_name, obj)
@@ -416,7 +453,7 @@ class BaseStateMachine(StateMachine):
 
     def __repr__(self) -> str:
         obj = getattr(self, self.model_attr_name)
-        return f'{self.__class__.__name__}[{obj.id}]'
+        return f"{self.__class__.__name__}[{obj.id}]"
 
     def __str__(self) -> str:
         return self.__repr__()

archivebox/workers/supervisord_util.py

@@ -1,4 +1,4 @@
-__package__ = 'archivebox.workers'
+__package__ = "archivebox.workers"
 
 import sys
 import time
@@ -8,7 +8,8 @@ import shutil
 import subprocess
 import shlex
 
-from typing import Dict, cast, Iterator
+from typing import cast
+from collections.abc import Iterator
 from pathlib import Path
 from functools import cache
 
@@ -34,6 +35,7 @@ _supervisord_proc = None
 def _shell_join(args: list[str]) -> str:
     return shlex.join(args)
 
+
 RUNNER_WORKER = {
     "name": "worker_runner",
     "command": _shell_join([sys.executable, "-m", "archivebox", "run", "--daemon"]),
@@ -54,7 +56,17 @@ RUNNER_WATCH_WORKER = lambda pidfile: {
 
 SERVER_WORKER = lambda host, port: {
     "name": "worker_daphne",
-    "command": _shell_join([sys.executable, "-m", "daphne", f"--bind={host}", f"--port={port}", "--application-close-timeout=600", "archivebox.core.asgi:application"]),
+    "command": _shell_join(
+        [
+            sys.executable,
+            "-m",
+            "daphne",
+            f"--bind={host}",
+            f"--port={port}",
+            "--application-close-timeout=600",
+            "archivebox.core.asgi:application",
+        ],
+    ),
     "autostart": "false",
     "autorestart": "true",
     "stdout_logfile": "logs/worker_daphne.log",
@@ -72,10 +84,12 @@ def RUNSERVER_WORKER(host: str, port: str, *, reload: bool, pidfile: str | None
     environment = ['ARCHIVEBOX_RUNSERVER="1"']
     if reload:
         assert pidfile, "RUNSERVER_WORKER requires a pidfile when reload=True"
-        environment.extend([
-            'ARCHIVEBOX_AUTORELOAD="1"',
-            f'ARCHIVEBOX_RUNSERVER_PIDFILE="{pidfile}"',
-        ])
+        environment.extend(
+            [
+                'ARCHIVEBOX_AUTORELOAD="1"',
+                f'ARCHIVEBOX_RUNSERVER_PIDFILE="{pidfile}"',
+            ],
+        )
 
     return {
         "name": "worker_runserver",
@@ -87,6 +101,7 @@ def RUNSERVER_WORKER(host: str, port: str, *, reload: bool, pidfile: str | None
         "redirect_stderr": "true",
     }
 
+
 def is_port_in_use(host: str, port: int) -> bool:
     """Check if a port is already in use."""
     try:
@@ -97,6 +112,7 @@ def is_port_in_use(host: str, port: int) -> bool:
     except OSError:
         return True
 
+
 @cache
 def get_sock_file():
     """Get the path to the supervisord socket file, symlinking to a shorter path if needed due to unix path length limits"""
@@ -106,17 +122,18 @@ def get_sock_file():
 
     return socket_file
 
+
 def follow(file, sleep_sec=0.1) -> Iterator[str]:
-    """ Yield each line from a file as they are written.
-    `sleep_sec` is the time to sleep after empty reads. """
-    line = ''
+    """Yield each line from a file as they are written.
+    `sleep_sec` is the time to sleep after empty reads."""
+    line = ""
     while True:
         tmp = file.readline()
         if tmp is not None and tmp != "":
             line += tmp
             if line.endswith("\n"):
                 yield line
-                line = ''
+                line = ""
         elif sleep_sec:
             time.sleep(sleep_sec)
 
@@ -127,7 +144,7 @@ def create_supervisord_config():
     CONFIG_FILE = SOCK_FILE.parent / CONFIG_FILE_NAME
     PID_FILE = SOCK_FILE.parent / PID_FILE_NAME
     LOG_FILE = CONSTANTS.LOGS_DIR / LOG_FILE_NAME
-    
+
     config_content = f"""
 [supervisord]
 nodaemon = true
@@ -156,22 +173,23 @@ files = {WORKERS_DIR}/*.conf
 """
     CONFIG_FILE.write_text(config_content)
     Path.mkdir(WORKERS_DIR, exist_ok=True, parents=True)
-    
-    (WORKERS_DIR / 'initial_startup.conf').write_text('')   # hides error about "no files found to include" when supervisord starts
+
+    (WORKERS_DIR / "initial_startup.conf").write_text("")  # hides error about "no files found to include" when supervisord starts
+
 
 def create_worker_config(daemon):
     """Create a supervisord worker config file for a given daemon"""
     SOCK_FILE = get_sock_file()
     WORKERS_DIR = SOCK_FILE.parent / WORKERS_DIR_NAME
-    
+
     Path.mkdir(WORKERS_DIR, exist_ok=True, parents=True)
-    
-    name = daemon['name']
+
+    name = daemon["name"]
     worker_conf = WORKERS_DIR / f"{name}.conf"
 
     worker_str = f"[program:{name}]\n"
     for key, value in daemon.items():
-        if key == 'name':
+        if key == "name":
             continue
         worker_str += f"{key}={value}\n"
     worker_str += "\n"
@@ -183,8 +201,11 @@ def get_existing_supervisord_process():
     SOCK_FILE = get_sock_file()
     try:
         transport = SupervisorTransport(None, None, f"unix://{SOCK_FILE}")
-        server = ServerProxy("http://localhost", transport=transport)       # user:pass@localhost doesn't work for some reason with unix://.sock, cant seem to silence CRIT no-auth warning
-        current_state = cast(Dict[str, int | str], server.supervisor.getState())
+        server = ServerProxy(
+            "http://localhost",
+            transport=transport,
+        )  # user:pass@localhost doesn't work for some reason with unix://.sock, cant seem to silence CRIT no-auth warning
+        current_state = cast(dict[str, int | str], server.supervisor.getState())
         if current_state["statename"] == "RUNNING":
             pid = server.supervisor.getPID()
             print(f"[🦸‍♂️] Supervisord connected (pid={pid}) via unix://{pretty_path(SOCK_FILE)}.")
@@ -195,6 +216,7 @@ def get_existing_supervisord_process():
         print(f"Error connecting to existing supervisord: {str(e)}")
         return None
 
+
 def stop_existing_supervisord_process():
     global _supervisord_proc
     SOCK_FILE = get_sock_file()
@@ -211,7 +233,7 @@ def stop_existing_supervisord_process():
                 except subprocess.TimeoutExpired:
                     _supervisord_proc.kill()
                     _supervisord_proc.wait(timeout=2)
-            except (BrokenPipeError, IOError):
+            except (BrokenPipeError, OSError):
                 pass
             finally:
                 _supervisord_proc = None
@@ -245,7 +267,7 @@ def stop_existing_supervisord_process():
                     pass
         except psutil.NoSuchProcess:
             pass
-        except (BrokenPipeError, IOError):
+        except (BrokenPipeError, OSError):
             pass
     finally:
         try:
@@ -255,6 +277,7 @@ def stop_existing_supervisord_process():
         except BaseException:
             pass
 
+
 def start_new_supervisord_process(daemonize=False):
     SOCK_FILE = get_sock_file()
     WORKERS_DIR = SOCK_FILE.parent / WORKERS_DIR_NAME
@@ -266,7 +289,7 @@ def start_new_supervisord_process(daemonize=False):
     pretty_log_path = pretty_path(LOG_FILE)
     print(f"    > Writing supervisord logs to: {pretty_log_path}")
     print(f"    > Writing task worker logs to: {pretty_log_path.replace('supervisord.log', 'worker_*.log')}")
-    print(f'    > Using supervisord config file: {pretty_path(CONFIG_FILE)}')
+    print(f"    > Using supervisord config file: {pretty_path(CONFIG_FILE)}")
     print(f"    > Using supervisord UNIX socket: {pretty_path(SOCK_FILE)}")
     print()
 
@@ -281,7 +304,7 @@ def start_new_supervisord_process(daemonize=False):
 
     # Open log file for supervisord output
     LOG_FILE.parent.mkdir(parents=True, exist_ok=True)
-    log_handle = open(LOG_FILE, 'a')
+    log_handle = open(LOG_FILE, "a")
 
     if daemonize:
         # Start supervisord in background (daemon mode)
@@ -329,7 +352,7 @@ def wait_for_supervisord_ready(max_wait_sec: float = 5.0, interval_sec: float =
 def get_or_create_supervisord_process(daemonize=False):
     SOCK_FILE = get_sock_file()
     WORKERS_DIR = SOCK_FILE.parent / WORKERS_DIR_NAME
-    
+
     supervisor = get_existing_supervisord_process()
     if supervisor is None:
         stop_existing_supervisord_process()
@@ -341,7 +364,7 @@ def get_or_create_supervisord_process(daemonize=False):
             if supervisor is not None:
                 print()
                 break
-            sys.stdout.write('.')
+            sys.stdout.write(".")
             sys.stdout.flush()
             time.sleep(0.1)
             supervisor = get_existing_supervisord_process()
@@ -351,10 +374,11 @@ def get_or_create_supervisord_process(daemonize=False):
     assert supervisor, "Failed to start supervisord or connect to it!"
     supervisor.getPID()  # make sure it doesn't throw an exception
 
-    (WORKERS_DIR / 'initial_startup.conf').unlink(missing_ok=True)
-    
+    (WORKERS_DIR / "initial_startup.conf").unlink(missing_ok=True)
+
     return supervisor
 
+
 def start_worker(supervisor, daemon, lazy=False):
     assert supervisor.getPID()
 
@@ -378,9 +402,9 @@ def start_worker(supervisor, daemon, lazy=False):
     for _ in range(25):
         procs = supervisor.getAllProcessInfo()
         for proc in procs:
-            if proc['name'] == daemon["name"]:
+            if proc["name"] == daemon["name"]:
                 # See process state diagram here: http://supervisord.org/subprocess.html
-                if proc['statename'] == 'RUNNING':
+                if proc["statename"] == "RUNNING":
                     print(f"     - Worker {daemon['name']}: already {proc['statename']} ({proc['description']})")
                     return proc
                 else:
@@ -403,6 +427,7 @@ def get_worker(supervisor, daemon_name):
         pass
     return None
 
+
 def stop_worker(supervisor, daemon_name):
     proc = get_worker(supervisor, daemon_name)
 
@@ -410,9 +435,9 @@ def stop_worker(supervisor, daemon_name):
         if not proc:
             # worker does not exist (was never running or configured in the first place)
             return True
-        
+
         # See process state diagram here: http://supervisord.org/subprocess.html
-        if proc['statename'] == 'STOPPED':
+        if proc["statename"] == "STOPPED":
             # worker was configured but has already stopped for some reason
             supervisor.removeProcessGroup(daemon_name)
             return True
@@ -439,12 +464,12 @@ def tail_worker_logs(log_path: str):
 
     try:
         with Live(table, refresh_per_second=1) as live:  # update 4 times a second to feel fluid
-            with open(log_path, 'r') as f:
+            with open(log_path) as f:
                 for line in follow(f):
-                    if '://' in line:
+                    if "://" in line:
                         live.console.print(f"Working on: {line.strip()}")
                     # table.add_row("123124234", line.strip())
-    except (KeyboardInterrupt, BrokenPipeError, IOError):
+    except (KeyboardInterrupt, BrokenPipeError, OSError):
         STDERR.print("\n[🛑] Got Ctrl+C, stopping gracefully...")
     except SystemExit:
         pass
@@ -479,7 +504,7 @@ def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None):
     file_handles = []
     for log_path in log_paths:
         try:
-            f = open(log_path, 'r')
+            f = open(log_path)
             # Seek to end - only show NEW logs from now on, not old logs
             f.seek(0, 2)  # Go to end
 
@@ -510,7 +535,7 @@ def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None):
                         break  # No more lines available in this file
                     had_output = True
                     # Strip ANSI codes if present (supervisord does this but just in case)
-                    line_clean = re.sub(r'\x1b\[[0-9;]*m', '', line.rstrip())
+                    line_clean = re.sub(r"\x1b\[[0-9;]*m", "", line.rstrip())
                     if line_clean:
                         print(line_clean)
 
@@ -518,7 +543,7 @@ def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None):
             if not had_output:
                 time.sleep(0.05)
 
-    except (KeyboardInterrupt, BrokenPipeError, IOError):
+    except (KeyboardInterrupt, BrokenPipeError, OSError):
         pass  # Let the caller handle the cleanup message
     except SystemExit:
         pass
@@ -530,45 +555,45 @@ def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None):
             except Exception:
                 pass
 
+
 def watch_worker(supervisor, daemon_name, interval=5):
     """loop continuously and monitor worker's health"""
     while True:
         proc = get_worker(supervisor, daemon_name)
         if not proc:
-            raise Exception("Worker dissapeared while running! " + daemon_name)
+            raise Exception("Worker disappeared while running! " + daemon_name)
 
-        if proc['statename'] == 'STOPPED':
+        if proc["statename"] == "STOPPED":
             return proc
 
-        if proc['statename'] == 'RUNNING':
+        if proc["statename"] == "RUNNING":
             time.sleep(1)
             continue
 
-        if proc['statename'] in ('STARTING', 'BACKOFF', 'FATAL', 'EXITED', 'STOPPING'):
-            print(f'[🦸‍♂️] WARNING: Worker {daemon_name} {proc["statename"]} {proc["description"]}')
+        if proc["statename"] in ("STARTING", "BACKOFF", "FATAL", "EXITED", "STOPPING"):
+            print(f"[🦸‍♂️] WARNING: Worker {daemon_name} {proc['statename']} {proc['description']}")
             time.sleep(interval)
             continue
 
 
-
-def start_server_workers(host='0.0.0.0', port='8000', daemonize=False, debug=False, reload=False, nothreading=False):
+def start_server_workers(host="0.0.0.0", port="8000", daemonize=False, debug=False, reload=False, nothreading=False):
     from archivebox.config.common import STORAGE_CONFIG
 
     supervisor = get_or_create_supervisord_process(daemonize=daemonize)
 
     if debug:
-        pidfile = str(STORAGE_CONFIG.TMP_DIR / 'runserver.pid') if reload else None
+        pidfile = str(STORAGE_CONFIG.TMP_DIR / "runserver.pid") if reload else None
         server_worker = RUNSERVER_WORKER(host=host, port=port, reload=reload, pidfile=pidfile, nothreading=nothreading)
         bg_workers: list[tuple[dict[str, str], bool]] = (
             [(RUNNER_WORKER, True), (RUNNER_WATCH_WORKER(pidfile), False)] if reload else [(RUNNER_WORKER, False)]
         )
-        log_files = ['logs/worker_runserver.log', 'logs/worker_runner.log']
+        log_files = ["logs/worker_runserver.log", "logs/worker_runner.log"]
         if reload:
-            log_files.insert(1, 'logs/worker_runner_watch.log')
+            log_files.insert(1, "logs/worker_runner_watch.log")
     else:
         server_worker = SERVER_WORKER(host=host, port=port)
         bg_workers = [(RUNNER_WORKER, False)]
-        log_files = ['logs/worker_daphne.log', 'logs/worker_runner.log']
+        log_files = ["logs/worker_daphne.log", "logs/worker_runner.log"]
 
     print()
     start_worker(supervisor, server_worker)
@@ -580,14 +605,14 @@ def start_server_workers(host='0.0.0.0', port='8000', daemonize=False, debug=Fal
     if not daemonize:
         try:
             # Tail worker logs while supervisord runs
-            sys.stdout.write('Tailing worker logs (Ctrl+C to stop)...\n\n')
+            sys.stdout.write("Tailing worker logs (Ctrl+C to stop)...\n\n")
             sys.stdout.flush()
             tail_multiple_worker_logs(
                 log_files=log_files,
                 follow=True,
                 proc=_supervisord_proc,  # Stop tailing when supervisord exits
             )
-        except (KeyboardInterrupt, BrokenPipeError, IOError):
+        except (KeyboardInterrupt, BrokenPipeError, OSError):
             STDERR.print("\n[🛑] Got Ctrl+C, stopping gracefully...")
         except SystemExit:
             pass
@@ -611,8 +636,8 @@ def start_cli_workers(watch=False):
                 _supervisord_proc.wait()
             else:
                 # Fallback to watching worker if no proc reference
-                watch_worker(supervisor, RUNNER_WORKER['name'])
-        except (KeyboardInterrupt, BrokenPipeError, IOError):
+                watch_worker(supervisor, RUNNER_WORKER["name"])
+        except (KeyboardInterrupt, BrokenPipeError, OSError):
             STDERR.print("\n[🛑] Got Ctrl+C, stopping gracefully...")
         except SystemExit:
             pass
@@ -632,14 +657,14 @@ def start_cli_workers(watch=False):
 #     pprint(worker)
 
 #     print("All processes started in background.")
-    
-    # Optionally you can block the main thread until an exit signal is received:
-    # try:
-    #     signal.pause()
-    # except KeyboardInterrupt:
-    #     pass
-    # finally:
-    #     stop_existing_supervisord_process()
+
+# Optionally you can block the main thread until an exit signal is received:
+# try:
+#     signal.pause()
+# except KeyboardInterrupt:
+#     pass
+# finally:
+#     stop_existing_supervisord_process()
 
 # if __name__ == "__main__":
 

archivebox/workers/tasks.py

@@ -8,7 +8,7 @@ NOTE: These functions do NOT start the runner. They assume it's already
 running via `archivebox server` or will be run inline by the CLI.
 """
 
-__package__ = 'archivebox.workers'
+__package__ = "archivebox.workers"
 
 from django.utils import timezone
 
@@ -25,7 +25,7 @@ def bg_add(add_kwargs: dict) -> int:
 
     # When called as background task, always run in background mode
     add_kwargs = add_kwargs.copy()
-    add_kwargs['bg'] = True
+    add_kwargs["bg"] = True
 
     _, result = add(**add_kwargs)
 
@@ -46,13 +46,13 @@ def bg_archive_snapshots(snapshots, kwargs: dict | None = None) -> int:
     # Queue snapshots by setting status to queued with immediate retry_at
     queued_count = 0
     for snapshot in snapshots:
-        if hasattr(snapshot, 'id'):
+        if hasattr(snapshot, "id"):
             Snapshot.objects.filter(id=snapshot.id).update(
                 status=Snapshot.StatusChoices.QUEUED,
                 retry_at=timezone.now(),
                 downloaded_at=None,
             )
-            crawl_id = getattr(snapshot, 'crawl_id', None)
+            crawl_id = getattr(snapshot, "crawl_id", None)
             if crawl_id:
                 Crawl.objects.filter(id=crawl_id).update(
                     status=Crawl.StatusChoices.QUEUED,
@@ -72,13 +72,13 @@ def bg_archive_snapshot(snapshot, overwrite: bool = False, methods: list | None
     from archivebox.core.models import Snapshot
     from archivebox.crawls.models import Crawl
 
-    if hasattr(snapshot, 'id'):
+    if hasattr(snapshot, "id"):
         Snapshot.objects.filter(id=snapshot.id).update(
             status=Snapshot.StatusChoices.QUEUED,
             retry_at=timezone.now(),
             downloaded_at=None,
         )
-        crawl_id = getattr(snapshot, 'crawl_id', None)
+        crawl_id = getattr(snapshot, "crawl_id", None)
         if crawl_id:
             Crawl.objects.filter(id=crawl_id).update(
                 status=Crawl.StatusChoices.QUEUED,