change plugins to have both a .register that runs at import and .ready that runs later

archivebox/plugantic/base_hook.py

@@ -1,14 +1,15 @@
 __package__ = 'archivebox.plugantic'
 
-import json
+import inspect
+from huey.api import TaskWrapper
+
+from pathlib import Path
 from typing import List, Literal
 from pydantic import BaseModel, ConfigDict, Field, computed_field
 
 
-HookType = Literal['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW']
-hook_type_names: List[HookType] = ['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW']
-
-
+HookType = Literal['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW', 'QUEUE']
+hook_type_names: List[HookType] = ['CONFIG', 'BINPROVIDER', 'BINARY', 'EXTRACTOR', 'REPLAYER', 'CHECK', 'ADMINDATAVIEW', 'QUEUE']
 
 class BaseHook(BaseModel):
     """
@@ -56,24 +57,37 @@ class BaseHook(BaseModel):
         validate_defaults=True,
         validate_assignment=False,
         revalidate_instances="subclass-instances",
+        ignored_types=(TaskWrapper, ),
     )
     
     # verbose_name: str = Field()
+    
+    is_registered: bool = False
+    is_ready: bool = False
 
 
     @computed_field
     @property
     def id(self) -> str:
         return self.__class__.__name__
-    
+
     @computed_field
     @property
     def hook_module(self) -> str:
+        """e.g. builtin_plugins.singlefile.apps.SinglefileConfigSet"""
         return f'{self.__module__}.{self.__class__.__name__}'
-    
+
+    @property
+    def plugin_module(self) -> str:
+        """e.g. builtin_plugins.singlefile"""
+        return f"{self.__module__}.{self.__class__.__name__}".split("archivebox.", 1)[-1].rsplit(".apps.", 1)[0]
+        
+    @computed_field
+    @property
+    def plugin_dir(self) -> Path:
+        return Path(inspect.getfile(self.__class__)).parent.resolve()
+
     hook_type: HookType = Field()
-    
-    
 
     def register(self, settings, parent_plugin=None):
         """Load a record of an installed hook into global Django settings.HOOKS at runtime."""
@@ -83,5 +97,20 @@ class BaseHook(BaseModel):
 
         # record installed hook in settings.HOOKS
         settings.HOOKS[self.id] = self
+        
+        if settings.HOOKS[self.id].is_registered:
+            raise Exception(f"Tried to run {self.hook_module}.register() but its already been called!")
+
+        settings.HOOKS[self.id].is_registered = True
 
         # print("REGISTERED HOOK:", self.hook_module)
+
+    def ready(self, settings):
+        """Runs any runtime code needed when AppConfig.ready() is called (after all models are imported)."""
+
+        assert self.id in settings.HOOKS, f"Tried to ready hook {self.hook_module} but it is not registered in settings.HOOKS."
+
+        if settings.HOOKS[self.id].is_ready:
+            raise Exception(f"Tried to run {self.hook_module}.ready() but its already been called!")
+
+        settings.HOOKS[self.id].is_ready = True

archivebox/plugantic/base_plugin.py

@@ -34,6 +34,9 @@ class BasePlugin(BaseModel):
     # All the hooks the plugin will install:
     hooks: List[InstanceOf[BaseHook]] = Field(default=[])
     
+    is_registered: bool = False
+    is_ready: bool = False
+    
     @computed_field
     @property
     def id(self) -> str:
@@ -81,7 +84,7 @@ class BasePlugin(BaseModel):
 
             def ready(self):
                 from django.conf import settings
-                plugin_self.register(settings)
+                plugin_self.ready(settings)
 
         return PluginAppConfig
 
@@ -97,9 +100,8 @@ class BasePlugin(BaseModel):
             hooks[hook.hook_type][hook.id] = hook
         return hooks
 
-
     def register(self, settings=None):
-        """Loads this plugin's configs, binaries, extractors, and replayers into global Django settings at runtime."""
+        """Loads this plugin's configs, binaries, extractors, and replayers into global Django settings at import time (before models are imported or any AppConfig.ready() are called)."""
 
         if settings is None:
             from django.conf import settings as django_settings
@@ -112,11 +114,34 @@ class BasePlugin(BaseModel):
         ### Mutate django.conf.settings... values in-place to include plugin-provided overrides
         settings.PLUGINS[self.id] = self
 
+        if settings.PLUGINS[self.id].is_registered:
+            raise Exception(f"Tried to run {self.plugin_module}.register() but its already been called!")
+
         for hook in self.hooks:
             hook.register(settings, parent_plugin=self)
 
+        settings.PLUGINS[self.id].is_registered = True
         # print('√ REGISTERED PLUGIN:', self.plugin_module)
 
+    def ready(self, settings=None):
+        """Runs any runtime code needed when AppConfig.ready() is called (after all models are imported)."""
+
+        if settings is None:
+            from django.conf import settings as django_settings
+            settings = django_settings
+
+        assert (
+            self.id in settings.PLUGINS and settings.PLUGINS[self.id].is_registered
+        ), f"Tried to run plugin.ready() for {self.plugin_module} but plugin is not yet registered in settings.PLUGINS."
+
+        if settings.PLUGINS[self.id].is_ready:
+            raise Exception(f"Tried to run {self.plugin_module}.ready() but its already been called!")
+
+        for hook in self.hooks:
+            hook.ready(settings)
+        
+        settings.PLUGINS[self.id].is_ready = True
+
     # @validate_call
     # def install_binaries(self) -> Self:
     #     new_binaries = []