much better tests and add page ui

archivebox/api/auth.py

@@ -13,7 +13,7 @@ from ninja.errors import HttpError
 
 
 def get_or_create_api_token(user):
-    from api.models import APIToken
+    from archivebox.api.models import APIToken
     
     if user and user.is_superuser:
         api_tokens = APIToken.objects.filter(created_by_id=user.pk, expires__gt=timezone.now())
@@ -32,7 +32,7 @@ def get_or_create_api_token(user):
 
 def auth_using_token(token, request: Optional[HttpRequest]=None) -> Optional[AbstractBaseUser]:
     """Given an API token string, check if a corresponding non-expired APIToken exists, and return its user"""
-    from api.models import APIToken        # lazy import model to avoid loading it at urls.py import time
+    from archivebox.api.models import APIToken        # lazy import model to avoid loading it at urls.py import time
     
     user = None
 

archivebox/api/migrations/0001_initial.py

@@ -0,0 +1,72 @@
+# Generated by hand on 2025-12-29
+# Creates APIToken and OutboundWebhook tables using raw SQL
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('auth', '0012_alter_user_first_name_max_length'),
+    ]
+
+    operations = [
+        migrations.RunSQL(
+            # Forward SQL
+            sql="""
+                -- Create api_apitoken table
+                CREATE TABLE IF NOT EXISTS api_apitoken (
+                    id TEXT PRIMARY KEY NOT NULL,
+                    created_at DATETIME NOT NULL,
+                    modified_at DATETIME NOT NULL,
+                    num_uses_succeeded INTEGER NOT NULL DEFAULT 0,
+                    num_uses_failed INTEGER NOT NULL DEFAULT 0,
+
+                    token VARCHAR(32) NOT NULL UNIQUE,
+                    label VARCHAR(64) NOT NULL DEFAULT '',
+                    notes TEXT NOT NULL DEFAULT '',
+                    expires DATETIME,
+
+                    created_by_id INTEGER NOT NULL,
+
+                    FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
+                );
+                CREATE INDEX IF NOT EXISTS api_apitoken_created_by_id_idx ON api_apitoken(created_by_id);
+                CREATE INDEX IF NOT EXISTS api_apitoken_token_idx ON api_apitoken(token);
+
+                -- Create api_outboundwebhook table
+                CREATE TABLE IF NOT EXISTS api_outboundwebhook (
+                    id TEXT PRIMARY KEY NOT NULL,
+                    created_at DATETIME NOT NULL,
+                    modified_at DATETIME NOT NULL,
+                    num_uses_succeeded INTEGER NOT NULL DEFAULT 0,
+                    num_uses_failed INTEGER NOT NULL DEFAULT 0,
+
+                    name VARCHAR(255) NOT NULL UNIQUE,
+                    signal VARCHAR(255) NOT NULL,
+                    ref VARCHAR(1024) NOT NULL,
+                    endpoint VARCHAR(2048) NOT NULL,
+                    headers TEXT NOT NULL DEFAULT '{}',
+                    enabled BOOLEAN NOT NULL DEFAULT 1,
+                    keep_last_response BOOLEAN NOT NULL DEFAULT 0,
+                    last_response TEXT,
+                    last_success DATETIME,
+                    last_error DATETIME,
+
+                    created_by_id INTEGER NOT NULL,
+
+                    FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
+                );
+                CREATE INDEX IF NOT EXISTS api_outboundwebhook_created_by_id_idx ON api_outboundwebhook(created_by_id);
+                CREATE INDEX IF NOT EXISTS api_outboundwebhook_name_idx ON api_outboundwebhook(name);
+                CREATE INDEX IF NOT EXISTS api_outboundwebhook_ref_idx ON api_outboundwebhook(ref);
+            """,
+            # Reverse SQL
+            reverse_sql="""
+                DROP TABLE IF EXISTS api_outboundwebhook;
+                DROP TABLE IF EXISTS api_apitoken;
+            """
+        ),
+    ]

archivebox/api/migrations/0001_squashed.py

@@ -1,74 +0,0 @@
-# Squashed migration: replaces 0001-0009
-# For fresh installs: creates final schema
-# For dev users with 0001-0009 applied: marked as applied (no-op)
-
-from uuid import uuid4
-from django.conf import settings
-from django.db import migrations, models
-import django.db.models.deletion
-
-import archivebox.api.models
-
-
-class Migration(migrations.Migration):
-
-    initial = True
-
-    replaces = [
-        ('api', '0001_initial'),
-        ('api', '0002_alter_apitoken_options'),
-        ('api', '0003_rename_user_apitoken_created_by_apitoken_abid_and_more'),
-        ('api', '0004_alter_apitoken_id_alter_apitoken_uuid'),
-        ('api', '0005_remove_apitoken_uuid_remove_outboundwebhook_uuid_and_more'),
-        ('api', '0006_remove_outboundwebhook_uuid_apitoken_id_and_more'),
-        ('api', '0007_alter_apitoken_created_by'),
-        ('api', '0008_alter_apitoken_created_alter_apitoken_created_by_and_more'),
-        ('api', '0009_rename_created_apitoken_created_at_and_more'),
-    ]
-
-    dependencies = [
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name='APIToken',
-            fields=[
-                ('id', models.UUIDField(default=uuid4, editable=False, primary_key=True, serialize=False, unique=True)),
-                ('created_by', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
-                ('modified_at', models.DateTimeField(auto_now=True)),
-                ('token', models.CharField(default=archivebox.api.models.generate_secret_token, max_length=32, unique=True)),
-                ('expires', models.DateTimeField(blank=True, null=True)),
-            ],
-            options={
-                'verbose_name': 'API Key',
-                'verbose_name_plural': 'API Keys',
-            },
-        ),
-        migrations.CreateModel(
-            name='OutboundWebhook',
-            fields=[
-                ('id', models.UUIDField(default=uuid4, editable=False, primary_key=True, serialize=False, unique=True)),
-                ('created_by', models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
-                ('created_at', models.DateTimeField(auto_now_add=True, db_index=True)),
-                ('modified_at', models.DateTimeField(auto_now=True)),
-                ('name', models.CharField(blank=True, default='', max_length=255)),
-                ('signal', models.CharField(choices=[], db_index=True, max_length=255)),
-                ('ref', models.CharField(db_index=True, max_length=255)),
-                ('endpoint', models.URLField(max_length=2083)),
-                ('headers', models.JSONField(blank=True, default=dict)),
-                ('auth_token', models.CharField(blank=True, default='', max_length=4000)),
-                ('enabled', models.BooleanField(db_index=True, default=True)),
-                ('keep_last_response', models.BooleanField(default=False)),
-                ('last_response', models.TextField(blank=True, default='')),
-                ('last_success', models.DateTimeField(blank=True, null=True)),
-                ('last_failure', models.DateTimeField(blank=True, null=True)),
-            ],
-            options={
-                'verbose_name': 'API Outbound Webhook',
-                'ordering': ['name', 'ref'],
-                'abstract': False,
-            },
-        ),
-    ]

archivebox/api/migrations/0002_alter_outboundwebhook_options_and_more.py

@@ -1,113 +0,0 @@
-# Generated by Django 6.0 on 2025-12-25 09:34
-
-import django.utils.timezone
-import signal_webhooks.fields
-import signal_webhooks.utils
-from archivebox import uuid_compat
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0001_squashed'),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.AlterModelOptions(
-            name='outboundwebhook',
-            options={'verbose_name': 'API Outbound Webhook'},
-        ),
-        migrations.AddField(
-            model_name='outboundwebhook',
-            name='created',
-            field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now, help_text='When the webhook was created.', verbose_name='created'),
-            preserve_default=False,
-        ),
-        migrations.AddField(
-            model_name='outboundwebhook',
-            name='updated',
-            field=models.DateTimeField(auto_now=True, help_text='When the webhook was last updated.', verbose_name='updated'),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='created_at',
-            field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
-        ),
-        migrations.AlterField(
-            model_name='apitoken',
-            name='id',
-            field=models.UUIDField(default=uuid_compat.uuid7, editable=False, primary_key=True, serialize=False, unique=True),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='auth_token',
-            field=signal_webhooks.fields.TokenField(blank=True, default='', help_text='Authentication token to use in an Authorization header.', max_length=8000, validators=[signal_webhooks.utils.decode_cipher_key], verbose_name='authentication token'),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='created_at',
-            field=models.DateTimeField(db_index=True, default=django.utils.timezone.now),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='enabled',
-            field=models.BooleanField(default=True, help_text='Is this webhook enabled?', verbose_name='enabled'),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='endpoint',
-            field=models.URLField(help_text='Target endpoint for this webhook.', max_length=2047, verbose_name='endpoint'),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='headers',
-            field=models.JSONField(blank=True, default=dict, help_text='Headers to send with the webhook request.', validators=[signal_webhooks.utils.is_dict], verbose_name='headers'),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='id',
-            field=models.UUIDField(default=uuid_compat.uuid7, editable=False, primary_key=True, serialize=False, unique=True),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='keep_last_response',
-            field=models.BooleanField(default=False, help_text='Should the webhook keep a log of the latest response it got?', verbose_name='keep last response'),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='last_failure',
-            field=models.DateTimeField(default=None, help_text='When the webhook last failed.', null=True, verbose_name='last failure'),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='last_response',
-            field=models.CharField(blank=True, default='', help_text='Latest response to this webhook.', max_length=8000, verbose_name='last response'),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='last_success',
-            field=models.DateTimeField(default=None, help_text='When the webhook last succeeded.', null=True, verbose_name='last success'),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='name',
-            field=models.CharField(db_index=True, help_text='Webhook name.', max_length=255, unique=True, verbose_name='name'),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='ref',
-            field=models.CharField(db_index=True, help_text='Dot import notation to the model the webhook is for.', max_length=1023, validators=[signal_webhooks.utils.model_from_reference], verbose_name='referenced model'),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='signal',
-            field=models.CharField(choices=[('CREATE', 'Create'), ('UPDATE', 'Update'), ('DELETE', 'Delete'), ('M2M', 'M2M changed'), ('CREATE_OR_UPDATE', 'Create or Update'), ('CREATE_OR_DELETE', 'Create or Delete'), ('CREATE_OR_M2M', 'Create or M2M changed'), ('UPDATE_OR_DELETE', 'Update or Delete'), ('UPDATE_OR_M2M', 'Update or M2M changed'), ('DELETE_OR_M2M', 'Delete or M2M changed'), ('CREATE_UPDATE_OR_DELETE', 'Create, Update or Delete'), ('CREATE_UPDATE_OR_M2M', 'Create, Update or M2M changed'), ('CREATE_DELETE_OR_M2M', 'Create, Delete or M2M changed'), ('UPDATE_DELETE_OR_M2M', 'Update, Delete or M2M changed'), ('CREATE_UPDATE_DELETE_OR_M2M', 'Create, Update or Delete, or M2M changed')], help_text='Signal the webhook fires to.', max_length=255, verbose_name='signal'),
-        ),
-        migrations.AddConstraint(
-            model_name='outboundwebhook',
-            constraint=models.UniqueConstraint(fields=('ref', 'endpoint'), name='prevent_duplicate_hooks_api_outboundwebhook'),
-        ),
-    ]

archivebox/api/migrations/0003_alter_apitoken_created_by_and_more.py

@@ -1,27 +0,0 @@
-# Generated by Django 6.0 on 2025-12-27 01:40
-
-import archivebox.core.models
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('api', '0002_alter_outboundwebhook_options_and_more'),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='apitoken',
-            name='created_by',
-            field=models.ForeignKey(default=archivebox.core.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
-        ),
-        migrations.AlterField(
-            model_name='outboundwebhook',
-            name='created_by',
-            field=models.ForeignKey(default=archivebox.core.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
-        ),
-    ]

archivebox/api/v1_api.py

@@ -37,12 +37,12 @@ html_description=f'''
 
 
 def register_urls(api: NinjaAPI) -> NinjaAPI:
-    # api.add_router('/auth/',     'api.v1_auth.router')
-    api.add_router('/core/',     'api.v1_core.router')
-    api.add_router('/crawls/',   'api.v1_crawls.router')
-    api.add_router('/cli/',      'api.v1_cli.router')
-    api.add_router('/workers/',  'api.v1_workers.router')
-    api.add_router('/machine/',  'api.v1_machine.router')
+    # api.add_router('/auth/',     'archivebox.api.v1_auth.router')
+    api.add_router('/core/',     'archivebox.api.v1_core.router')
+    api.add_router('/crawls/',   'archivebox.api.v1_crawls.router')
+    api.add_router('/cli/',      'archivebox.api.v1_cli.router')
+    api.add_router('/workers/',  'archivebox.api.v1_workers.router')
+    api.add_router('/machine/',  'archivebox.api.v1_machine.router')
     return api
 
 

archivebox/api/v1_core.py

@@ -67,6 +67,7 @@ class MinimalArchiveResultSchema(Schema):
     retry_at: datetime | None
     plugin: str
     hook_name: str
+    process_id: UUID | None
     cmd_version: str | None
     cmd: list[str] | None
     pwd: str | None
@@ -121,6 +122,7 @@ class ArchiveResultFilterSchema(FilterSchema):
     output_str: Optional[str] = Field(None, q='output_str__icontains')
     plugin: Optional[str] = Field(None, q='plugin__icontains')
     hook_name: Optional[str] = Field(None, q='hook_name__icontains')
+    process_id: Optional[str] = Field(None, q='process__id__startswith')
     cmd: Optional[str] = Field(None, q='cmd__0__icontains')
     pwd: Optional[str] = Field(None, q='pwd__icontains')
     cmd_version: Optional[str] = Field(None, q='cmd_version')
@@ -290,7 +292,7 @@ def get_any(request, id: str):
             pass
 
     try:
-        from api.v1_crawls import get_crawl
+        from archivebox.api.v1_crawls import get_crawl
         response = get_crawl(request, id)
         if response:
             return redirect(f"/api/v1/{response._meta.app_label}/{response._meta.model_name}/{response.id}?{request.META['QUERY_STRING']}")

archivebox/api/v1_workers.py

@@ -95,7 +95,7 @@ class OrchestratorSchema(Schema):
 def get_orchestrator(request):
     """Get the orchestrator status and all worker queues."""
     from archivebox.workers.orchestrator import Orchestrator
-    from workers.worker import CrawlWorker, SnapshotWorker, ArchiveResultWorker
+    from archivebox.workers.worker import CrawlWorker, SnapshotWorker, ArchiveResultWorker
 
     orchestrator = Orchestrator()
 
@@ -120,7 +120,7 @@ def get_orchestrator(request):
 @router.get("/workers", response=List[WorkerSchema], url_name="get_workers")
 def get_workers(request):
     """List all worker types and their current status."""
-    from workers.worker import CrawlWorker, SnapshotWorker, ArchiveResultWorker
+    from archivebox.workers.worker import CrawlWorker, SnapshotWorker, ArchiveResultWorker
 
     # Create temporary instances to query their queues
     return [
@@ -133,7 +133,7 @@ def get_workers(request):
 @router.get("/worker/{worker_name}", response=WorkerSchema, url_name="get_worker")
 def get_worker(request, worker_name: str):
     """Get status and queue for a specific worker type."""
-    from workers.worker import WORKER_TYPES
+    from archivebox.workers.worker import WORKER_TYPES
 
     if worker_name not in WORKER_TYPES:
         from ninja.errors import HttpError
@@ -146,7 +146,7 @@ def get_worker(request, worker_name: str):
 @router.get("/worker/{worker_name}/queue", response=List[QueueItemSchema], url_name="get_worker_queue")
 def get_worker_queue(request, worker_name: str, limit: int = 100):
     """Get the current queue for a specific worker type."""
-    from workers.worker import WORKER_TYPES
+    from archivebox.workers.worker import WORKER_TYPES
 
     if worker_name not in WORKER_TYPES:
         from ninja.errors import HttpError