+
+ {progress_text}
+
+ {str(result.id)[:8]}...
+ Version: {version}
+ PWD: {result.pwd or '-'}
+ {full_output}
+ {cmd_str}
+ {}:{}', source_file, contents)
-
+
return format_html('See URLs here: {}', obj.uri, obj.uri)
@@ -78,7 +218,37 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
search_fields = ('id', 'created_by__username', 'max_depth', 'label', 'notes', 'seed_id', 'schedule_id', 'status', 'seed__uri')
readonly_fields = ('created_at', 'modified_at', 'snapshots', 'seed_urls_editor')
- fields = ('label', 'notes', 'seed_urls_editor', 'config', 'status', 'retry_at', 'max_depth', 'seed', 'schedule', 'created_by', 'created_at', 'modified_at', 'snapshots')
+
+ fieldsets = (
+ ('URLs', {
+ 'fields': ('seed_urls_editor',),
+ 'classes': ('card', 'wide'),
+ }),
+ ('Info', {
+ 'fields': ('label', 'notes'),
+ 'classes': ('card',),
+ }),
+ ('Settings', {
+ 'fields': ('max_depth', 'config'),
+ 'classes': ('card',),
+ }),
+ ('Status', {
+ 'fields': ('status', 'retry_at'),
+ 'classes': ('card',),
+ }),
+ ('Relations', {
+ 'fields': ('seed', 'schedule', 'created_by'),
+ 'classes': ('card',),
+ }),
+ ('Timestamps', {
+ 'fields': ('created_at', 'modified_at'),
+ 'classes': ('card',),
+ }),
+ ('Snapshots', {
+ 'fields': ('snapshots',),
+ 'classes': ('card', 'wide'),
+ }),
+ )
list_filter = ('max_depth', 'seed', 'schedule', 'created_by', 'status', 'retry_at')
ordering = ['-created_at', '-retry_at']
@@ -90,6 +260,16 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
def recrawl(self, request, obj):
"""Duplicate this crawl as a new crawl with the same seed and settings."""
from django.utils import timezone
+ from django.shortcuts import redirect
+
+ # Validate seed has a URI (required for crawl to start)
+ if not obj.seed:
+ messages.error(request, 'Cannot recrawl: original crawl has no seed.')
+ return redirect('admin:crawls_crawl_change', obj.id)
+
+ if not obj.seed.uri:
+ messages.error(request, 'Cannot recrawl: seed has no URI.')
+ return redirect('admin:crawls_crawl_change', obj.id)
new_crawl = Crawl.objects.create(
seed=obj.seed,
@@ -110,8 +290,6 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
f'It will start processing shortly.'
)
- # Redirect to the new crawl's change page
- from django.shortcuts import redirect
return redirect('admin:crawls_crawl_change', new_crawl.id)
def get_urls(self):
@@ -133,7 +311,8 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
except Crawl.DoesNotExist:
return JsonResponse({'success': False, 'error': 'Crawl not found'}, status=404)
- if not (crawl.seed and crawl.seed.uri and crawl.seed.uri.startswith('file:///data/')):
+ source_file = crawl.seed.get_file_path() if crawl.seed else None
+ if not source_file:
return JsonResponse({'success': False, 'error': 'Seed is not a local file'}, status=400)
try:
@@ -142,8 +321,6 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
except json.JSONDecodeError:
return JsonResponse({'success': False, 'error': 'Invalid JSON'}, status=400)
- source_file = DATA_DIR / crawl.seed.uri.replace('file:///data/', '', 1)
-
try:
# Ensure parent directory exists
source_file.parent.mkdir(parents=True, exist_ok=True)
@@ -156,10 +333,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
return obj.snapshot_set.count()
def snapshots(self, obj):
- return format_html_join('