alex/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-01-04 09:55:33 +10:00
Code Issues Packages Projects Releases Wiki Activity

working migrations again

Browse Source
This commit is contained in:
Nick Sweeting
2025-12-31 16:19:50 -08:00
parent 6521e7ddda
commit 1c7b0cb2d3
1 changed files with 144 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

277
archivebox/core/migrations/0023_upgrade_to_0_9_0.py
View File

@@ -16,17 +16,16 @@ def upgrade_core_tables(apps, schema_editor):
"""Upgrade core tables from v0.7.2 or v0.8.6rc0 to v0.9.0.""" """Upgrade core tables from v0.7.2 or v0.8.6rc0 to v0.9.0."""
cursor = connection.cursor() cursor = connection.cursor()
# Check if core_archiveresult table exists AND has data # Check if core_archiveresult table exists
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='core_archiveresult'") cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='core_archiveresult'")
if not cursor.fetchone(): if not cursor.fetchone():
# Fresh install - no migration needed, tables will be created by later migrations # Fresh install - no migration needed, tables will be created by later migrations
return return
# Check if table has any rows (fresh install has empty tables from CREATE TABLE IF NOT EXISTS) # Check if table has any rows
cursor.execute("SELECT COUNT(*) FROM core_archiveresult") cursor.execute("SELECT COUNT(*) FROM core_archiveresult")
if cursor.fetchone()[0] == 0: row_count = cursor.fetchone()[0]
# Fresh install with empty tables - skip migration has_data = row_count > 0
return
# Detect which version we're migrating from # Detect which version we're migrating from
archiveresult_cols = get_table_columns('core_archiveresult') archiveresult_cols = get_table_columns('core_archiveresult')
@@ -71,45 +70,46 @@ def upgrade_core_tables(apps, schema_editor):
); );
""") """)
if has_uuid and not has_abid: if has_data:
# Migrating from v0.7.2 (has uuid, minimal fields) if has_uuid and not has_abid:
print('Migrating ArchiveResult from v0.7.2 schema...') # Migrating from v0.7.2 (has uuid, minimal fields)
cursor.execute(""" print('Migrating ArchiveResult from v0.7.2 schema...')
INSERT OR IGNORE INTO core_archiveresult_new ( cursor.execute("""
id, uuid, created_at, modified_at, snapshot_id, plugin, INSERT OR IGNORE INTO core_archiveresult_new (
cmd, pwd, cmd_version, start_ts, end_ts, status, output_str id, uuid, created_at, modified_at, snapshot_id, plugin,
) cmd, pwd, cmd_version, start_ts, end_ts, status, output_str
SELECT )
id, uuid, SELECT
COALESCE(start_ts, CURRENT_TIMESTAMP) as created_at, id, uuid,
COALESCE(end_ts, start_ts, CURRENT_TIMESTAMP) as modified_at, COALESCE(start_ts, CURRENT_TIMESTAMP) as created_at,
snapshot_id, COALESCE(end_ts, start_ts, CURRENT_TIMESTAMP) as modified_at,
COALESCE(extractor, '') as plugin, snapshot_id,
cmd, pwd, cmd_version, COALESCE(extractor, '') as plugin,
start_ts, end_ts, status, cmd, pwd, cmd_version,
COALESCE(output, '') as output_str start_ts, end_ts, status,
FROM core_archiveresult; COALESCE(output, '') as output_str
""") FROM core_archiveresult;
elif has_abid and not has_uuid: """)
# Migrating from v0.8.6rc0 (has abid, full fields) elif has_abid and not has_uuid:
print('Migrating ArchiveResult from v0.8.6rc0 schema...') # Migrating from v0.8.6rc0 (has abid, full fields)
cursor.execute(""" print('Migrating ArchiveResult from v0.8.6rc0 schema...')
INSERT OR IGNORE INTO core_archiveresult_new ( cursor.execute("""
id, uuid, created_at, modified_at, snapshot_id, plugin, INSERT OR IGNORE INTO core_archiveresult_new (
cmd, pwd, cmd_version, start_ts, end_ts, status, retry_at, output_str id, uuid, created_at, modified_at, snapshot_id, plugin,
) cmd, pwd, cmd_version, start_ts, end_ts, status, retry_at, output_str
SELECT )
id, abid as uuid, SELECT
created_at, modified_at, id, abid as uuid,
snapshot_id, created_at, modified_at,
COALESCE(extractor, '') as plugin, snapshot_id,
cmd, pwd, cmd_version, COALESCE(extractor, '') as plugin,
start_ts, end_ts, status, retry_at, cmd, pwd, cmd_version,
COALESCE(output, '') as output_str start_ts, end_ts, status, retry_at,
FROM core_archiveresult; COALESCE(output, '') as output_str
""") FROM core_archiveresult;
else: """)
print(f'Warning: Unexpected schema - has_uuid={has_uuid}, has_abid={has_abid}') else:
print(f'Warning: Unexpected schema - has_uuid={has_uuid}, has_abid={has_abid}')
cursor.execute("DROP TABLE IF EXISTS core_archiveresult;") cursor.execute("DROP TABLE IF EXISTS core_archiveresult;")
cursor.execute("ALTER TABLE core_archiveresult_new RENAME TO core_archiveresult;") cursor.execute("ALTER TABLE core_archiveresult_new RENAME TO core_archiveresult;")
@@ -160,49 +160,54 @@ def upgrade_core_tables(apps, schema_editor):
# Check if core_snapshot exists (it should) # Check if core_snapshot exists (it should)
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='core_snapshot'") cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='core_snapshot'")
if cursor.fetchone(): if cursor.fetchone():
# Detect which version we're migrating from # Check if table has any rows
snapshot_cols = get_table_columns('core_snapshot') cursor.execute("SELECT COUNT(*) FROM core_snapshot")
has_added = 'added' in snapshot_cols snapshot_has_data = cursor.fetchone()[0] > 0
has_bookmarked_at = 'bookmarked_at' in snapshot_cols
if has_added and not has_bookmarked_at: if snapshot_has_data:
# Migrating from v0.7.2 (has added/updated, no bookmarked_at/created_at/modified_at) # Detect which version we're migrating from
print('Migrating Snapshot from v0.7.2 schema...') snapshot_cols = get_table_columns('core_snapshot')
cursor.execute(""" has_added = 'added' in snapshot_cols
INSERT OR IGNORE INTO core_snapshot_new ( has_bookmarked_at = 'bookmarked_at' in snapshot_cols
id, url, timestamp, title, bookmarked_at, created_at, modified_at
)
SELECT
id, url, timestamp, title,
COALESCE(added, CURRENT_TIMESTAMP) as bookmarked_at,
COALESCE(added, CURRENT_TIMESTAMP) as created_at,
COALESCE(updated, added, CURRENT_TIMESTAMP) as modified_at
FROM core_snapshot;
""")
elif has_bookmarked_at and not has_added:
# Migrating from v0.8.6rc0 (already has bookmarked_at/created_at/modified_at)
print('Migrating Snapshot from v0.8.6rc0 schema...')
# Check what fields exist
has_status = 'status' in snapshot_cols
has_retry_at = 'retry_at' in snapshot_cols
has_crawl_id = 'crawl_id' in snapshot_cols
# Build column list based on what exists if has_added and not has_bookmarked_at:
cols = ['id', 'url', 'timestamp', 'title', 'bookmarked_at', 'created_at', 'modified_at', 'downloaded_at'] # Migrating from v0.7.2 (has added/updated, no bookmarked_at/created_at/modified_at)
if has_crawl_id: print('Migrating Snapshot from v0.7.2 schema...')
cols.append('crawl_id') cursor.execute("""
if has_status: INSERT OR IGNORE INTO core_snapshot_new (
cols.append('status') id, url, timestamp, title, bookmarked_at, created_at, modified_at
if has_retry_at: )
cols.append('retry_at') SELECT
id, url, timestamp, title,
COALESCE(added, CURRENT_TIMESTAMP) as bookmarked_at,
COALESCE(added, CURRENT_TIMESTAMP) as created_at,
COALESCE(updated, added, CURRENT_TIMESTAMP) as modified_at
FROM core_snapshot;
""")
elif has_bookmarked_at and not has_added:
# Migrating from v0.8.6rc0 (already has bookmarked_at/created_at/modified_at)
print('Migrating Snapshot from v0.8.6rc0 schema...')
# Check what fields exist
has_status = 'status' in snapshot_cols
has_retry_at = 'retry_at' in snapshot_cols
has_crawl_id = 'crawl_id' in snapshot_cols
cursor.execute(f""" # Build column list based on what exists
INSERT OR IGNORE INTO core_snapshot_new ({', '.join(cols)}) cols = ['id', 'url', 'timestamp', 'title', 'bookmarked_at', 'created_at', 'modified_at', 'downloaded_at']
SELECT {', '.join(cols)} if has_crawl_id:
FROM core_snapshot; cols.append('crawl_id')
""") if has_status:
else: cols.append('status')
print(f'Warning: Unexpected Snapshot schema - has_added={has_added}, has_bookmarked_at={has_bookmarked_at}') if has_retry_at:
cols.append('retry_at')
cursor.execute(f"""
INSERT OR IGNORE INTO core_snapshot_new ({', '.join(cols)})
SELECT {', '.join(cols)}
FROM core_snapshot;
""")
else:
print(f'Warning: Unexpected Snapshot schema - has_added={has_added}, has_bookmarked_at={has_bookmarked_at}')
cursor.execute("DROP TABLE IF EXISTS core_snapshot;") cursor.execute("DROP TABLE IF EXISTS core_snapshot;")
cursor.execute("ALTER TABLE core_snapshot_new RENAME TO core_snapshot;") cursor.execute("ALTER TABLE core_snapshot_new RENAME TO core_snapshot;")
@@ -237,58 +242,63 @@ def upgrade_core_tables(apps, schema_editor):
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='core_tag'") cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='core_tag'")
if cursor.fetchone(): if cursor.fetchone():
tag_cols = get_table_columns('core_tag') # Check if table has any rows
cursor.execute("PRAGMA table_info(core_tag)") cursor.execute("SELECT COUNT(*) FROM core_tag")
tag_id_type = None tag_has_data = cursor.fetchone()[0] > 0
for row in cursor.fetchall():
if row[1] == 'id': # row[1] is column name
tag_id_type = row[2] # row[2] is type
break
if tag_id_type and 'char' in tag_id_type.lower(): if tag_has_data:
# v0.8.6rc0: Tag IDs are UUIDs, need to convert to INTEGER tag_cols = get_table_columns('core_tag')
print('Converting Tag IDs from UUID to INTEGER...') cursor.execute("PRAGMA table_info(core_tag)")
tag_id_type = None
for row in cursor.fetchall():
if row[1] == 'id': # row[1] is column name
tag_id_type = row[2] # row[2] is type
break
# Get all tags with their UUIDs if tag_id_type and 'char' in tag_id_type.lower():
cursor.execute("SELECT id, name, slug, created_at, modified_at, created_by_id FROM core_tag ORDER BY name") # v0.8.6rc0: Tag IDs are UUIDs, need to convert to INTEGER
tags = cursor.fetchall() print('Converting Tag IDs from UUID to INTEGER...')
# Create mapping from old UUID to new INTEGER ID # Get all tags with their UUIDs
uuid_to_int_map = {} cursor.execute("SELECT id, name, slug, created_at, modified_at, created_by_id FROM core_tag ORDER BY name")
for i, tag in enumerate(tags, start=1): tags = cursor.fetchall()
old_id, name, slug, created_at, modified_at, created_by_id = tag
uuid_to_int_map[old_id] = i # Create mapping from old UUID to new INTEGER ID
# Insert with new INTEGER ID uuid_to_int_map = {}
for i, tag in enumerate(tags, start=1):
old_id, name, slug, created_at, modified_at, created_by_id = tag
uuid_to_int_map[old_id] = i
# Insert with new INTEGER ID
cursor.execute("""
INSERT OR IGNORE INTO core_tag_new (id, name, slug, created_at, modified_at, created_by_id)
VALUES (?, ?, ?, ?, ?, ?)
""", (i, name, slug, created_at, modified_at, created_by_id))
# Update snapshot_tags to use new INTEGER IDs
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='core_snapshot_tags'")
if cursor.fetchone():
cursor.execute("SELECT id, snapshot_id, tag_id FROM core_snapshot_tags")
snapshot_tags = cursor.fetchall()
# Delete old entries
cursor.execute("DELETE FROM core_snapshot_tags")
# Re-insert with new integer tag IDs
for st_id, snapshot_id, old_tag_id in snapshot_tags:
new_tag_id = uuid_to_int_map.get(old_tag_id)
if new_tag_id:
cursor.execute("""
INSERT OR IGNORE INTO core_snapshot_tags (id, snapshot_id, tag_id)
VALUES (?, ?, ?)
""", (st_id, snapshot_id, new_tag_id))
else:
# v0.7.2: Tag IDs are already INTEGER
print('Migrating Tag from v0.7.2 schema...')
cursor.execute(""" cursor.execute("""
INSERT OR IGNORE INTO core_tag_new (id, name, slug, created_at, modified_at, created_by_id) INSERT OR IGNORE INTO core_tag_new (id, name, slug)
VALUES (?, ?, ?, ?, ?, ?) SELECT id, name, slug
""", (i, name, slug, created_at, modified_at, created_by_id)) FROM core_tag;
""")
# Update snapshot_tags to use new INTEGER IDs
cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='core_snapshot_tags'")
if cursor.fetchone():
cursor.execute("SELECT id, snapshot_id, tag_id FROM core_snapshot_tags")
snapshot_tags = cursor.fetchall()
# Delete old entries
cursor.execute("DELETE FROM core_snapshot_tags")
# Re-insert with new integer tag IDs
for st_id, snapshot_id, old_tag_id in snapshot_tags:
new_tag_id = uuid_to_int_map.get(old_tag_id)
if new_tag_id:
cursor.execute("""
INSERT OR IGNORE INTO core_snapshot_tags (id, snapshot_id, tag_id)
VALUES (?, ?, ?)
""", (st_id, snapshot_id, new_tag_id))
else:
# v0.7.2: Tag IDs are already INTEGER
print('Migrating Tag from v0.7.2 schema...')
cursor.execute("""
INSERT OR IGNORE INTO core_tag_new (id, name, slug)
SELECT id, name, slug
FROM core_tag;
""")
cursor.execute("DROP TABLE IF EXISTS core_tag;") cursor.execute("DROP TABLE IF EXISTS core_tag;")
cursor.execute("ALTER TABLE core_tag_new RENAME TO core_tag;") cursor.execute("ALTER TABLE core_tag_new RENAME TO core_tag;")
@@ -297,7 +307,8 @@ def upgrade_core_tables(apps, schema_editor):
cursor.execute("CREATE INDEX IF NOT EXISTS core_tag_created_at_idx ON core_tag(created_at);") cursor.execute("CREATE INDEX IF NOT EXISTS core_tag_created_at_idx ON core_tag(created_at);")
cursor.execute("CREATE INDEX IF NOT EXISTS core_tag_created_by_id_idx ON core_tag(created_by_id);") cursor.execute("CREATE INDEX IF NOT EXISTS core_tag_created_by_id_idx ON core_tag(created_by_id);")
print('✓ Core tables upgraded to v0.9.0') if has_data:
print('✓ Core tables upgraded to v0.9.0')
class Migration(migrations.Migration): class Migration(migrations.Migration):