alex/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-04-05 15:27:53 +10:00
Code Issues Packages Projects Releases Wiki Activity

remove .old_id entirely and make ABID generation only happen once on initial save

Browse Source
This commit is contained in:
Nick Sweeting
2024-09-04 16:40:15 -07:00
parent 22a6666bab
commit 68a39b7392
11 changed files with 242 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
archivebox/core/admin.py
View File

@@ -240,8 +240,8 @@ class ArchiveResultInline(admin.TabularInline):
# fk_name = 'snapshot'
extra = 0
sort_fields = ('end_ts', 'extractor', 'output', 'status', 'cmd_version')
readonly_fields = ('result_id', 'completed', 'extractor', 'command', 'version')
fields = ('id', 'start_ts', 'end_ts', *readonly_fields, 'cmd', 'cmd_version', 'pwd', 'created_by', 'status', 'output')
readonly_fields = ('id', 'result_id', 'completed', 'command', 'version')
fields = ('start_ts', 'end_ts', *readonly_fields, 'extractor', 'cmd', 'cmd_version', 'pwd', 'created_by', 'status', 'output')
# exclude = ('id',)
ordering = ('end_ts',)
show_change_link = True
@@ -273,7 +273,7 @@ class ArchiveResultInline(admin.TabularInline):
snapshot = self.get_parent_object_from_request(request)
# import ipdb; ipdb.set_trace()
formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget()
# formset.form.base_fields['id'].widget = formset.form.base_fields['id'].hidden_widget()
# default values for new entries
formset.form.base_fields['status'].initial = 'succeeded'
@@ -351,13 +351,13 @@ class SnapshotActionForm(ActionForm):
@admin.register(Snapshot, site=archivebox_admin)
class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
list_display = ('added', 'title_str', 'files', 'size', 'url_str')
sort_fields = ('title_str', 'url_str', 'added')
readonly_fields = ('tags_str', 'timestamp', 'admin_actions', 'status_info', 'bookmarked', 'created', 'added', 'updated', 'modified', 'API', 'link_dir')
search_fields = ('id', 'url', 'abid', 'old_id', 'timestamp', 'title', 'tags__name')
list_filter = ('added', 'updated', 'archiveresult__status', 'created_by', 'tags__name')
list_display = ('created', 'title_str', 'files', 'size', 'url_str')
sort_fields = ('title_str', 'url_str', 'created')
readonly_fields = ('tags_str', 'timestamp', 'admin_actions', 'status_info', 'bookmarked', 'created', 'created', 'updated', 'modified', 'API', 'link_dir')
search_fields = ('id', 'url', 'abid', 'timestamp', 'title', 'tags__name')
list_filter = ('created', 'updated', 'archiveresult__status', 'created_by', 'tags__name')
fields = ('url', 'created_by', 'title',*readonly_fields)
ordering = ['-added']
ordering = ['-created']
actions = ['add_tags', 'remove_tags', 'update_titles', 'update_snapshots', 'resnapshot_snapshot', 'overwrite_snapshots', 'delete_snapshots']
inlines = [TagInline, ArchiveResultInline]
list_per_page = min(max(5, CONFIG.SNAPSHOTS_PER_PAGE), 5000)
@@ -391,12 +391,6 @@ class SnapshotAdmin(SearchResultsAdminMixin, ABIDModelAdmin):
except (Snapshot.DoesNotExist, ValidationError):
pass
try:
snapshot = snapshot or Snapshot.objects.get(old_id=object_id)
except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned, ValidationError):
pass
if snapshot:
object_id = str(snapshot.id)
@@ -690,7 +684,7 @@ class ArchiveResultAdmin(ABIDModelAdmin):
list_display = ('start_ts', 'snapshot_info', 'tags_str', 'extractor', 'cmd_str', 'status', 'output_str')
sort_fields = ('start_ts', 'extractor', 'status')
readonly_fields = ('cmd_str', 'snapshot_info', 'tags_str', 'created', 'modified', 'API', 'output_summary')
search_fields = ('id', 'old_id', 'abid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
search_fields = ('id', 'abid', 'snapshot__url', 'extractor', 'output', 'cmd_version', 'cmd', 'snapshot__timestamp')
fields = ('snapshot', 'extractor', 'status', 'output', 'pwd', 'start_ts', 'end_ts', 'created_by', 'cmd_version', 'cmd', *readonly_fields)
autocomplete_fields = ['snapshot']

52
archivebox/core/migrations/0070_alter_archiveresult_created_by_alter_snapshot_added_and_more.py Normal file
View File

@@ -0,0 +1,52 @@
# Generated by Django 5.1 on 2024-09-04 09:00
import abid_utils.models
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0069_alter_archiveresult_created_alter_snapshot_added_and_more'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.AlterField(
model_name='archiveresult',
name='created_by',
field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, related_name='archiveresult_set', to=settings.AUTH_USER_MODEL),
),
migrations.AlterField(
model_name='snapshot',
name='added',
field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
),
migrations.AlterField(
model_name='snapshot',
name='created',
field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
),
migrations.AlterField(
model_name='snapshot',
name='created_by',
field=models.ForeignKey(default=abid_utils.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, related_name='snapshot_set', to=settings.AUTH_USER_MODEL),
),
migrations.AlterField(
model_name='snapshot',
name='id',
field=models.UUIDField(default=None, primary_key=True, serialize=False, unique=True),
),
migrations.AlterField(
model_name='snapshot',
name='old_id',
field=models.UUIDField(default=None, editable=False, unique=True),
),
migrations.AlterField(
model_name='tag',
name='created',
field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
),
]

65
archivebox/core/migrations/0071_remove_archiveresult_old_id_remove_snapshot_old_id_and_more.py Normal file
View File

@@ -0,0 +1,65 @@
# Generated by Django 5.1 on 2024-09-04 23:23
import abid_utils.models
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0070_alter_archiveresult_created_by_alter_snapshot_added_and_more'),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.RemoveField(
model_name='archiveresult',
name='old_id',
),
migrations.RemoveField(
model_name='snapshot',
name='old_id',
),
migrations.RemoveField(
model_name='tag',
name='old_id',
),
migrations.AlterField(
model_name='archiveresult',
name='created',
field=abid_utils.models.AutoDateTimeField(db_index=True, default=None),
),
migrations.AlterField(
model_name='archiveresult',
name='created_by',
field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='archiveresult_set', to=settings.AUTH_USER_MODEL),
),
migrations.AlterField(
model_name='archiveresult',
name='id',
field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
),
migrations.AlterField(
model_name='snapshot',
name='created_by',
field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='snapshot_set', to=settings.AUTH_USER_MODEL),
),
migrations.AlterField(
model_name='snapshot',
name='id',
field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
),
migrations.AlterField(
model_name='tag',
name='created_by',
field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, related_name='tag_set', to=settings.AUTH_USER_MODEL),
),
migrations.AlterField(
model_name='tag',
name='id',
field=models.UUIDField(default=None, editable=False, primary_key=True, serialize=False, unique=True, verbose_name='ID'),
),
]

34
archivebox/core/models.py
View File

@@ -54,13 +54,15 @@ class Tag(ABIDModel):
abid_ts_src = 'self.created'
abid_uri_src = 'self.slug'
abid_subtype_src = '"03"'
abid_rand_src = 'self.old_id'
abid_rand_src = 'self.id'
old_id = models.BigIntegerField(unique=True, default=rand_int_id, serialize=False, verbose_name='Old ID') # legacy PK
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False, unique=True)
id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
abid = ABIDField(prefix=abid_prefix)
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='tag_set')
created = AutoDateTimeField(default=None, null=False, db_index=True)
modified = models.DateTimeField(auto_now=True)
name = models.CharField(unique=True, blank=False, max_length=100)
slug = models.SlugField(unique=True, blank=False, max_length=100, editable=False)
# slug is autoset on save from name, never set it manually
@@ -74,10 +76,6 @@ class Tag(ABIDModel):
def __str__(self):
return self.name
# @property
# def old_id(self):
# return self.id
def slugify(self, tag, i=None):
slug = slugify(tag)
if i is not None:
@@ -133,16 +131,15 @@ class SnapshotManager(models.Manager):
class Snapshot(ABIDModel):
abid_prefix = 'snp_'
abid_ts_src = 'self.added'
abid_ts_src = 'self.created'
abid_uri_src = 'self.url'
abid_subtype_src = '"01"'
abid_rand_src = 'self.old_id'
abid_rand_src = 'self.id'
old_id = models.UUIDField(default=None, null=False, editable=False, unique=True) # legacy pk
id = models.UUIDField(default=None, null=False, primary_key=True, editable=True, unique=True)
id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
abid = ABIDField(prefix=abid_prefix)
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, related_name='snapshot_set')
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='snapshot_set')
created = AutoDateTimeField(default=None, null=False, db_index=True)
modified = models.DateTimeField(auto_now=True)
@@ -161,7 +158,6 @@ class Snapshot(ABIDModel):
objects = SnapshotManager()
def __repr__(self) -> str:
title = (self.title_stripped or '-')[:64]
return f'[{self.timestamp}] {self.url[:64]} ({title})'
@@ -414,7 +410,7 @@ class ArchiveResult(ABIDModel):
abid_ts_src = 'self.snapshot.added'
abid_uri_src = 'self.snapshot.url'
abid_subtype_src = 'self.extractor'
abid_rand_src = 'self.old_id'
abid_rand_src = 'self.id'
EXTRACTOR_CHOICES = (
('htmltotext', 'htmltotext'),
@@ -438,13 +434,11 @@ class ArchiveResult(ABIDModel):
("skipped", "skipped")
]
old_id = models.BigIntegerField(default=rand_int_id, serialize=False, verbose_name='Old ID')
id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True, unique=True, verbose_name='ID')
id = models.UUIDField(primary_key=True, default=None, null=False, editable=False, unique=True, verbose_name='ID')
abid = ABIDField(prefix=abid_prefix)
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, related_name='archiveresult_set')
created = AutoDateTimeField(default=timezone.now, db_index=True)
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=None, null=False, related_name='archiveresult_set')
created = AutoDateTimeField(default=None, null=False, db_index=True)
modified = models.DateTimeField(auto_now=True)
snapshot = models.ForeignKey(Snapshot, on_delete=models.CASCADE, to_field='id', db_column='snapshot_id')

10
archivebox/core/views.py
View File

@@ -285,17 +285,17 @@ class SnapshotView(View):
# ulid = slug.split('_', 1)[-1]
# try:
# try:
# snapshot = snapshot or Snapshot.objects.get(Q(abid=ulid) | Q(id=ulid) | Q(old_id=ulid))
# snapshot = snapshot or Snapshot.objects.get(Q(abid=ulid) | Q(id=ulid))
# except Snapshot.DoesNotExist:
# pass
# try:
# snapshot = Snapshot.objects.get(Q(abid__startswith=slug) | Q(abid__startswith=Snapshot.abid_prefix + slug) | Q(id__startswith=slug) | Q(old_id__startswith=slug))
# snapshot = Snapshot.objects.get(Q(abid__startswith=slug) | Q(abid__startswith=Snapshot.abid_prefix + slug) | Q(id__startswith=slug))
# except (Snapshot.DoesNotExist, Snapshot.MultipleObjectsReturned):
# pass
# try:
# snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id) | Q(old_id__icontains=snapshot_id))
# snapshot = snapshot or Snapshot.objects.get(Q(abid__icontains=snapshot_id) | Q(id__icontains=snapshot_id))
# except Snapshot.DoesNotExist:
# pass
# return redirect(f'/archive/{snapshot.timestamp}/index.html')
@@ -308,7 +308,7 @@ class SnapshotView(View):
# try exact match on full url / ABID first
snapshot = Snapshot.objects.get(
Q(url='http://' + path) | Q(url='https://' + path) | Q(id__startswith=path)
| Q(abid__icontains=path) | Q(id__icontains=path) | Q(old_id__icontains=path)
| Q(abid__icontains=path) | Q(id__icontains=path)
)
except Snapshot.DoesNotExist:
# fall back to match on exact base_url
@@ -352,7 +352,7 @@ class SnapshotView(View):
)
for snap in Snapshot.objects.filter(
Q(url__startswith='http://' + base_url(path)) | Q(url__startswith='https://' + base_url(path))
| Q(abid__icontains=path) | Q(id__icontains=path) | Q(old_id__icontains=path)
| Q(abid__icontains=path) | Q(id__icontains=path)
).only('url', 'timestamp', 'title', 'added').order_by('-added')
)
return HttpResponse(