ArchiveBox/archivebox/crawls/migrations/0001_initial.py

# Generated by hand on 2025-12-29
# Creates Crawl and CrawlSchedule tables using raw SQL

from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
import django.core.validators
from django.conf import settings
from archivebox.uuid_compat import uuid7
from archivebox.base_models.models import get_or_create_system_user_pk


class Migration(migrations.Migration):

    initial = True

    dependencies = [
        ('auth', '0012_alter_user_first_name_max_length'),
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
        migrations.SeparateDatabaseAndState(
            database_operations=[
                migrations.RunSQL(
                    sql="""
                -- Create crawls_crawlschedule table first (circular FK will be added later)
                CREATE TABLE IF NOT EXISTS crawls_crawlschedule (
                    id TEXT PRIMARY KEY NOT NULL,
                    created_at DATETIME NOT NULL,
                    modified_at DATETIME NOT NULL,
                    num_uses_succeeded INTEGER NOT NULL DEFAULT 0,
                    num_uses_failed INTEGER NOT NULL DEFAULT 0,

                    schedule VARCHAR(64) NOT NULL,
                    is_enabled BOOLEAN NOT NULL DEFAULT 1,
                    label VARCHAR(64) NOT NULL DEFAULT '',
                    notes TEXT NOT NULL DEFAULT '',

                    template_id TEXT NOT NULL,
                    created_by_id INTEGER NOT NULL,

                    FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE
                );
                CREATE INDEX IF NOT EXISTS crawls_crawlschedule_created_at_idx ON crawls_crawlschedule(created_at);
                CREATE INDEX IF NOT EXISTS crawls_crawlschedule_created_by_id_idx ON crawls_crawlschedule(created_by_id);
                CREATE INDEX IF NOT EXISTS crawls_crawlschedule_template_id_idx ON crawls_crawlschedule(template_id);

                -- Create crawls_crawl table
                CREATE TABLE IF NOT EXISTS crawls_crawl (
                    id TEXT PRIMARY KEY NOT NULL,
                    created_at DATETIME NOT NULL,
                    modified_at DATETIME NOT NULL,
                    num_uses_succeeded INTEGER NOT NULL DEFAULT 0,
                    num_uses_failed INTEGER NOT NULL DEFAULT 0,

                    urls TEXT NOT NULL,
                    config TEXT,
                    max_depth INTEGER NOT NULL DEFAULT 0,
                    tags_str VARCHAR(1024) NOT NULL DEFAULT '',
                    persona_id TEXT,
                    label VARCHAR(64) NOT NULL DEFAULT '',
                    notes TEXT NOT NULL DEFAULT '',
                    output_dir VARCHAR(512) NOT NULL DEFAULT '',

                    status VARCHAR(15) NOT NULL DEFAULT 'queued',
                    retry_at DATETIME,
                    created_by_id INTEGER NOT NULL,
                    schedule_id TEXT,

                    FOREIGN KEY (created_by_id) REFERENCES auth_user(id) ON DELETE CASCADE,
                    FOREIGN KEY (schedule_id) REFERENCES crawls_crawlschedule(id) ON DELETE SET NULL
                );
                CREATE INDEX IF NOT EXISTS crawls_crawl_status_idx ON crawls_crawl(status);
                CREATE INDEX IF NOT EXISTS crawls_crawl_retry_at_idx ON crawls_crawl(retry_at);
                CREATE INDEX IF NOT EXISTS crawls_crawl_created_at_idx ON crawls_crawl(created_at);
                CREATE INDEX IF NOT EXISTS crawls_crawl_created_by_id_idx ON crawls_crawl(created_by_id);
                CREATE INDEX IF NOT EXISTS crawls_crawl_schedule_id_idx ON crawls_crawl(schedule_id);
                    """,
                    reverse_sql="""
                DROP TABLE IF EXISTS crawls_crawl;
                DROP TABLE IF EXISTS crawls_crawlschedule;
                    """
                ),
            ],
            state_operations=[
                migrations.CreateModel(
                    name='CrawlSchedule',
                    fields=[
                        ('id', models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
                        ('created_at', models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
                        ('modified_at', models.DateTimeField(auto_now=True)),
                        ('num_uses_succeeded', models.PositiveIntegerField(default=0)),
                        ('num_uses_failed', models.PositiveIntegerField(default=0)),
                        ('schedule', models.CharField(max_length=64)),
                        ('is_enabled', models.BooleanField(default=True)),
                        ('label', models.CharField(blank=True, default='', max_length=64)),
                        ('notes', models.TextField(blank=True, default='')),
                        ('created_by', models.ForeignKey(default=get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
                    ],
                    options={
                        'verbose_name': 'Scheduled Crawl',
                        'verbose_name_plural': 'Scheduled Crawls',
                        'app_label': 'crawls',
                    },
                ),
                migrations.CreateModel(
                    name='Crawl',
                    fields=[
                        ('id', models.UUIDField(default=uuid7, editable=False, primary_key=True, serialize=False, unique=True)),
                        ('created_at', models.DateTimeField(db_index=True, default=django.utils.timezone.now)),
                        ('modified_at', models.DateTimeField(auto_now=True)),
                        ('num_uses_succeeded', models.PositiveIntegerField(default=0)),
                        ('num_uses_failed', models.PositiveIntegerField(default=0)),
                        ('urls', models.TextField(help_text='Newline-separated list of URLs to crawl')),
                        ('config', models.JSONField(blank=True, default=dict, null=True)),
                        ('max_depth', models.PositiveSmallIntegerField(default=0, validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(4)])),
                        ('tags_str', models.CharField(blank=True, default='', max_length=1024)),
                        ('persona_id', models.UUIDField(blank=True, null=True)),
                        ('label', models.CharField(blank=True, default='', max_length=64)),
                        ('notes', models.TextField(blank=True, default='')),
                        ('output_dir', models.CharField(blank=True, default='', max_length=512)),
                        ('status', models.CharField(choices=[('queued', 'Queued'), ('started', 'Started'), ('sealed', 'Sealed')], db_index=True, default='queued', max_length=15)),
                        ('retry_at', models.DateTimeField(blank=True, db_index=True, default=django.utils.timezone.now, null=True)),
                        ('created_by', models.ForeignKey(default=get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
                        ('schedule', models.ForeignKey(blank=True, editable=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='crawls.crawlschedule')),
                    ],
                    options={
                        'verbose_name': 'Crawl',
                        'verbose_name_plural': 'Crawls',
                        'app_label': 'crawls',
                    },
                ),
                migrations.AddField(
                    model_name='crawlschedule',
                    name='template',
                    field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='crawls.crawl'),
                ),
            ],
        ),
    ]