/save_seed_contents/',
- self.admin_site.admin_view(self.save_seed_contents_view),
- name='crawls_crawl_save_seed_contents'),
- ]
- return custom_urls + urls
-
- def save_seed_contents_view(self, request, object_id):
- """Handle saving seed file contents via AJAX."""
- if request.method != 'POST':
- return JsonResponse({'success': False, 'error': 'POST required'}, status=405)
-
- try:
- crawl = Crawl.objects.get(pk=object_id)
- except Crawl.DoesNotExist:
- return JsonResponse({'success': False, 'error': 'Crawl not found'}, status=404)
-
- source_file = crawl.seed.get_file_path() if crawl.seed else None
- if not source_file:
- return JsonResponse({'success': False, 'error': 'Seed is not a local file'}, status=400)
-
- try:
- data = json.loads(request.body)
- contents = data.get('contents', '')
- except json.JSONDecodeError:
- return JsonResponse({'success': False, 'error': 'Invalid JSON'}, status=400)
-
- try:
- # Ensure parent directory exists
- source_file.parent.mkdir(parents=True, exist_ok=True)
- source_file.write_text(contents)
- return JsonResponse({'success': True, 'message': f'Saved {len(contents)} bytes to {source_file.name}'})
- except Exception as e:
- return JsonResponse({'success': False, 'error': str(e)}, status=500)
-
def num_snapshots(self, obj):
return obj.snapshot_set.count()
@@ -341,163 +225,68 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
return mark_safe('None')
return format_html('{}', obj.schedule.admin_change_url, obj.schedule)
- @admin.display(description='Seed', ordering='seed')
- def seed_str(self, obj):
- if not obj.seed:
- return mark_safe('None')
- return format_html('{}', obj.seed.admin_change_url, obj.seed)
+ @admin.display(description='URLs', ordering='urls')
+ def urls_preview(self, obj):
+ first_url = obj.get_urls_list()[0] if obj.get_urls_list() else ''
+ return first_url[:80] + '...' if len(first_url) > 80 else first_url
@admin.display(description='URLs')
- def seed_urls_editor(self, obj):
- """Combined editor showing seed URL and file contents."""
- widget_id = f'seed_urls_{obj.pk}'
-
- # Get the seed URI (or use urls field if no seed)
- seed_uri = ''
- if obj.seed and obj.seed.uri:
- seed_uri = obj.seed.uri
- elif obj.urls:
- seed_uri = obj.urls
+ def urls_editor(self, obj):
+ """Editor for crawl URLs."""
+ widget_id = f'crawl_urls_{obj.pk}'
# Check if it's a local file we can edit
- source_file = obj.seed.get_file_path() if obj.seed else None
+ source_file = obj.get_file_path()
is_file = source_file is not None
- contents = ""
+ file_contents = ""
error = None
if is_file and source_file:
try:
- contents = source_file.read_text().strip()
+ file_contents = source_file.read_text().strip()
except Exception as e:
error = f'Error reading {source_file}: {e}'
# Escape for safe HTML embedding
- escaped_uri = seed_uri.replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"')
- escaped_contents = (contents or '').replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"')
+ escaped_urls = (obj.urls or '').replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"')
+ escaped_file_contents = file_contents.replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"')
# Count lines for auto-expand logic
- line_count = len(contents.split('\n')) if contents else 0
- uri_rows = min(max(1, seed_uri.count('\n') + 1), 3)
+ line_count = len((obj.urls or '').split('\n'))
+ file_line_count = len(file_contents.split('\n')) if file_contents else 0
+ uri_rows = min(max(3, line_count), 10)
html = f'''
'''
return mark_safe(html)
@@ -507,7 +296,7 @@ class CrawlAdmin(ConfigEditorMixin, BaseModelAdmin):
class CrawlScheduleAdmin(BaseModelAdmin):
list_display = ('id', 'created_at', 'created_by', 'label', 'notes', 'template_str', 'crawls', 'num_crawls', 'num_snapshots')
sort_fields = ('id', 'created_at', 'created_by', 'label', 'notes', 'template_str')
- search_fields = ('id', 'created_by__username', 'label', 'notes', 'schedule_id', 'template_id', 'template__seed__uri')
+ search_fields = ('id', 'created_by__username', 'label', 'notes', 'schedule_id', 'template_id', 'template__urls')
readonly_fields = ('created_at', 'modified_at', 'crawls', 'snapshots')
@@ -561,6 +350,5 @@ class CrawlScheduleAdmin(BaseModelAdmin):
def register_admin(admin_site):
- admin_site.register(Seed, SeedAdmin)
admin_site.register(Crawl, CrawlAdmin)
admin_site.register(CrawlSchedule, CrawlScheduleAdmin)
diff --git a/archivebox/crawls/migrations/0002_drop_seed_model.py b/archivebox/crawls/migrations/0002_drop_seed_model.py
new file mode 100644
index 00000000..f0a66af5
--- /dev/null
+++ b/archivebox/crawls/migrations/0002_drop_seed_model.py
@@ -0,0 +1,61 @@
+# Generated by Django 6.0 on 2025-12-25 09:34
+
+import archivebox.base_models.models
+import django.db.models.deletion
+import pathlib
+import uuid
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('crawls', '0001_initial'),
+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name='crawl',
+ name='seed',
+ ),
+ migrations.AddField(
+ model_name='crawl',
+ name='extractor',
+ field=models.CharField(default='auto', help_text='Parser for reading URLs (auto, html, json, rss, etc)', max_length=32),
+ ),
+ migrations.AlterField(
+ model_name='crawl',
+ name='created_by',
+ field=models.ForeignKey(default=archivebox.base_models.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+ ),
+ migrations.AlterField(
+ model_name='crawl',
+ name='id',
+ field=models.UUIDField(default=uuid.uuid7, editable=False, primary_key=True, serialize=False, unique=True),
+ ),
+ migrations.AlterField(
+ model_name='crawl',
+ name='output_dir',
+ field=models.FilePathField(blank=True, default='', path=pathlib.PurePosixPath('/Users/squash/Local/Code/archiveboxes/archivebox-nue/data/archive')),
+ ),
+ migrations.AlterField(
+ model_name='crawl',
+ name='urls',
+ field=models.TextField(help_text='Newline-separated list of URLs to crawl'),
+ ),
+ migrations.AlterField(
+ model_name='crawlschedule',
+ name='created_by',
+ field=models.ForeignKey(default=archivebox.base_models.models.get_or_create_system_user_pk, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+ ),
+ migrations.AlterField(
+ model_name='crawlschedule',
+ name='id',
+ field=models.UUIDField(default=uuid.uuid7, editable=False, primary_key=True, serialize=False, unique=True),
+ ),
+ migrations.DeleteModel(
+ name='Seed',
+ ),
+ ]
diff --git a/archivebox/crawls/models.py b/archivebox/crawls/models.py
index 4bd00328..263869fe 100644
--- a/archivebox/crawls/models.py
+++ b/archivebox/crawls/models.py
@@ -20,91 +20,6 @@ if TYPE_CHECKING:
from core.models import Snapshot, ArchiveResult
-class Seed(ModelWithOutputDir, ModelWithConfig, ModelWithNotes, ModelWithHealthStats):
- id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
- created_at = models.DateTimeField(default=timezone.now, db_index=True)
- created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
- modified_at = models.DateTimeField(auto_now=True)
-
- uri = models.URLField(max_length=2048)
- extractor = models.CharField(default='auto', max_length=32)
- tags_str = models.CharField(max_length=255, null=False, blank=True, default='')
- label = models.CharField(max_length=255, null=False, blank=True, default='')
- config = models.JSONField(default=dict)
- output_dir = models.FilePathField(path=settings.ARCHIVE_DIR, null=False, blank=True, default='')
- notes = models.TextField(blank=True, null=False, default='')
-
- crawl_set: models.Manager['Crawl']
-
- class Meta:
- verbose_name = 'Seed'
- verbose_name_plural = 'Seeds'
- unique_together = (('created_by', 'uri', 'extractor'), ('created_by', 'label'))
-
- def __str__(self):
- return f'[{self.id}] {self.uri[:64]}'
-
- def save(self, *args, **kwargs):
- is_new = self._state.adding
- super().save(*args, **kwargs)
- if is_new:
- from archivebox.misc.logging_util import log_worker_event
- log_worker_event(
- worker_type='DB',
- event='Created Seed',
- indent_level=0,
- metadata={
- 'id': str(self.id),
- 'uri': str(self.uri)[:64],
- 'extractor': self.extractor,
- 'label': self.label or None,
- },
- )
-
- @classmethod
- def from_file(cls, source_file: Path, label: str = '', parser: str = 'auto', tag: str = '', created_by=None, config=None):
- # Use absolute path for file:// URLs so extractors can find the files
- source_path = str(source_file.resolve())
- seed, _ = cls.objects.get_or_create(
- label=label or source_file.name, uri=f'file://{source_path}',
- created_by_id=getattr(created_by, 'pk', created_by) or get_or_create_system_user_pk(),
- extractor=parser, tags_str=tag, config=config or {},
- )
- return seed
-
- @property
- def source_type(self):
- return self.uri.split('://', 1)[0].lower()
-
- @property
- def api_url(self) -> str:
- return reverse_lazy('api-1:get_seed', args=[self.id])
-
- def get_file_path(self) -> Path | None:
- """
- Get the filesystem path for file:// URIs.
- Handles both old format (file:///data/...) and new format (file:///absolute/path).
- Returns None if URI is not a file:// URI.
- """
- if not self.uri.startswith('file://'):
- return None
-
- # Remove file:// prefix
- path_str = self.uri.replace('file://', '', 1)
-
- # Handle old format: file:///data/... -> DATA_DIR/...
- if path_str.startswith('/data/'):
- return CONSTANTS.DATA_DIR / path_str.replace('/data/', '', 1)
-
- # Handle new format: file:///absolute/path
- return Path(path_str)
-
- @property
- def snapshot_set(self) -> QuerySet['Snapshot']:
- from core.models import Snapshot
- return Snapshot.objects.filter(crawl_id__in=self.crawl_set.values_list('pk', flat=True))
-
-
class CrawlSchedule(ModelWithSerializers, ModelWithNotes, ModelWithHealthStats):
id = models.UUIDField(primary_key=True, default=uuid7, editable=False, unique=True)
created_at = models.DateTimeField(default=timezone.now, db_index=True)
@@ -124,14 +39,15 @@ class CrawlSchedule(ModelWithSerializers, ModelWithNotes, ModelWithHealthStats):
verbose_name_plural = 'Scheduled Crawls'
def __str__(self) -> str:
- return f'[{self.id}] {self.template.seed.uri[:64] if self.template and self.template.seed else ""} @ {self.schedule}'
+ urls_preview = self.template.urls[:64] if self.template and self.template.urls else ""
+ return f'[{self.id}] {urls_preview} @ {self.schedule}'
@property
def api_url(self) -> str:
return reverse_lazy('api-1:get_any', args=[self.id])
def save(self, *args, **kwargs):
- self.label = self.label or (self.template.seed.label if self.template and self.template.seed else '')
+ self.label = self.label or (self.template.label if self.template else '')
super().save(*args, **kwargs)
if self.template:
self.template.schedule = self
@@ -144,8 +60,8 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
created_by = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, default=get_or_create_system_user_pk, null=False)
modified_at = models.DateTimeField(auto_now=True)
- seed = models.ForeignKey(Seed, on_delete=models.PROTECT, related_name='crawl_set', null=False, blank=False)
- urls = models.TextField(blank=True, null=False, default='')
+ urls = models.TextField(blank=False, null=False, help_text='Newline-separated list of URLs to crawl')
+ extractor = models.CharField(default='auto', max_length=32, help_text='Parser for reading URLs (auto, html, json, rss, etc)')
config = models.JSONField(default=dict)
max_depth = models.PositiveSmallIntegerField(default=0, validators=[MinValueValidator(0), MaxValueValidator(4)])
tags_str = models.CharField(max_length=1024, blank=True, null=False, default='')
@@ -171,31 +87,40 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
verbose_name_plural = 'Crawls'
def __str__(self):
- return f'[{self.id}] {self.seed.uri[:64] if self.seed else ""}'
+ first_url = self.get_urls_list()[0] if self.get_urls_list() else ''
+ return f'[{self.id}] {first_url[:64]}'
def save(self, *args, **kwargs):
is_new = self._state.adding
super().save(*args, **kwargs)
if is_new:
from archivebox.misc.logging_util import log_worker_event
+ first_url = self.get_urls_list()[0] if self.get_urls_list() else ''
log_worker_event(
worker_type='DB',
event='Created Crawl',
indent_level=1,
metadata={
'id': str(self.id),
- 'seed_uri': str(self.seed.uri)[:64] if self.seed else None,
+ 'first_url': first_url[:64],
'max_depth': self.max_depth,
'status': self.status,
},
)
@classmethod
- def from_seed(cls, seed: Seed, max_depth: int = 0, persona: str = 'Default', tags_str: str = '', config=None, created_by=None):
- crawl, _ = cls.objects.get_or_create(
- seed=seed, max_depth=max_depth, tags_str=tags_str or seed.tags_str,
- config=seed.config or config or {},
- created_by_id=getattr(created_by, 'pk', created_by) or seed.created_by_id,
+ def from_file(cls, source_file: Path, max_depth: int = 0, label: str = '', extractor: str = 'auto',
+ tags_str: str = '', config=None, created_by=None):
+ """Create a crawl from a file containing URLs."""
+ urls_content = source_file.read_text()
+ crawl = cls.objects.create(
+ urls=urls_content,
+ extractor=extractor,
+ max_depth=max_depth,
+ tags_str=tags_str,
+ label=label or source_file.name,
+ config=config or {},
+ created_by_id=getattr(created_by, 'pk', created_by) or get_or_create_system_user_pk(),
)
return crawl
@@ -203,14 +128,47 @@ class Crawl(ModelWithOutputDir, ModelWithConfig, ModelWithHealthStats, ModelWith
def api_url(self) -> str:
return reverse_lazy('api-1:get_crawl', args=[self.id])
+ def get_urls_list(self) -> list[str]:
+ """Get list of URLs from urls field, filtering out comments and empty lines."""
+ if not self.urls:
+ return []
+ return [
+ url.strip()
+ for url in self.urls.split('\n')
+ if url.strip() and not url.strip().startswith('#')
+ ]
+
+ def get_file_path(self) -> Path | None:
+ """
+ Get filesystem path if this crawl references a local file.
+ Checks if the first URL is a file:// URI.
+ """
+ urls = self.get_urls_list()
+ if not urls:
+ return None
+
+ first_url = urls[0]
+ if not first_url.startswith('file://'):
+ return None
+
+ # Remove file:// prefix
+ path_str = first_url.replace('file://', '', 1)
+ return Path(path_str)
+
def create_root_snapshot(self) -> 'Snapshot':
from core.models import Snapshot
+
+ first_url = self.get_urls_list()[0] if self.get_urls_list() else None
+ if not first_url:
+ raise ValueError(f'Crawl {self.id} has no URLs to create root snapshot from')
+
try:
- return Snapshot.objects.get(crawl=self, url=self.seed.uri)
+ return Snapshot.objects.get(crawl=self, url=first_url)
except Snapshot.DoesNotExist:
pass
+
root_snapshot, _ = Snapshot.objects.update_or_create(
- crawl=self, url=self.seed.uri,
+ crawl=self, url=first_url,
defaults={
'status': Snapshot.INITIAL_STATE,
'retry_at': timezone.now(),
diff --git a/archivebox/crawls/statemachines.py b/archivebox/crawls/statemachines.py
index a71cd010..45cb62fc 100644
--- a/archivebox/crawls/statemachines.py
+++ b/archivebox/crawls/statemachines.py
@@ -42,11 +42,12 @@ class CrawlMachine(StateMachine, strict_states=True):
return self.__repr__()
def can_start(self) -> bool:
- if not self.crawl.seed:
- print(f'[red]β οΈ Crawl {self.crawl.id} cannot start: no seed[/red]')
+ if not self.crawl.urls:
+ print(f'[red]β οΈ Crawl {self.crawl.id} cannot start: no URLs[/red]')
return False
- if not self.crawl.seed.uri:
- print(f'[red]β οΈ Crawl {self.crawl.id} cannot start: seed has no URI[/red]')
+ urls_list = self.crawl.get_urls_list()
+ if not urls_list:
+ print(f'[red]β οΈ Crawl {self.crawl.id} cannot start: no valid URLs in urls field[/red]')
return False
return True
@@ -121,13 +122,14 @@ class CrawlMachine(StateMachine, strict_states=True):
output_dir.mkdir(parents=True, exist_ok=True)
# Run all on_Crawl hooks
+ first_url = self.crawl.get_urls_list()[0] if self.crawl.get_urls_list() else ''
results = run_hooks(
event_name='Crawl',
output_dir=output_dir,
timeout=60,
- config_objects=[self.crawl, self.crawl.seed] if self.crawl.seed else [self.crawl],
+ config_objects=[self.crawl],
crawl_id=str(self.crawl.id),
- seed_uri=self.crawl.seed.uri if self.crawl.seed else '',
+ seed_uri=first_url,
)
# Process hook results - parse JSONL output and create DB objects
diff --git a/archivebox/logs/errors.log b/archivebox/logs/errors.log
deleted file mode 100644
index 715cf9d3..00000000
--- a/archivebox/logs/errors.log
+++ /dev/null
@@ -1,2 +0,0 @@
-
-> /Users/squash/Local/Code/archiveboxes/archivebox-nue/archivebox/cli/archivebox_init.py --force; TS=2025-12-25__08:03:12 VERSION=0.9.0rc1 IN_DOCKER=False IS_TTY=False
diff --git a/archivebox/machine/migrations/0002_alter_dependency_bin_name_and_more.py b/archivebox/machine/migrations/0002_alter_dependency_bin_name_and_more.py
new file mode 100644
index 00000000..6df9a423
--- /dev/null
+++ b/archivebox/machine/migrations/0002_alter_dependency_bin_name_and_more.py
@@ -0,0 +1,65 @@
+# Generated by Django 6.0 on 2025-12-25 09:34
+
+import django.db.models.deletion
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('machine', '0001_squashed'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='dependency',
+ name='bin_name',
+ field=models.CharField(db_index=True, help_text='Binary executable name (e.g., wget, yt-dlp, chromium)', max_length=63, unique=True),
+ ),
+ migrations.AlterField(
+ model_name='dependency',
+ name='bin_providers',
+ field=models.CharField(default='*', help_text='Comma-separated list of allowed providers: apt,brew,pip,npm,gem,nix,custom or * for any', max_length=127),
+ ),
+ migrations.AlterField(
+ model_name='dependency',
+ name='config',
+ field=models.JSONField(blank=True, default=dict, help_text='JSON map of env var config to use during install'),
+ ),
+ migrations.AlterField(
+ model_name='dependency',
+ name='custom_cmds',
+ field=models.JSONField(blank=True, default=dict, help_text="JSON map of provider -> custom install command (e.g., {'apt': 'apt install -y wget'})"),
+ ),
+ migrations.AlterField(
+ model_name='dependency',
+ name='id',
+ field=models.UUIDField(default=uuid.uuid7, editable=False, primary_key=True, serialize=False, unique=True),
+ ),
+ migrations.AlterField(
+ model_name='installedbinary',
+ name='dependency',
+ field=models.ForeignKey(blank=True, help_text='The Dependency this binary satisfies', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='installedbinary_set', to='machine.dependency'),
+ ),
+ migrations.AlterField(
+ model_name='installedbinary',
+ name='id',
+ field=models.UUIDField(default=uuid.uuid7, editable=False, primary_key=True, serialize=False, unique=True),
+ ),
+ migrations.AlterField(
+ model_name='machine',
+ name='config',
+ field=models.JSONField(blank=True, default=dict, help_text='Machine-specific config overrides (e.g., resolved binary paths like WGET_BINARY)'),
+ ),
+ migrations.AlterField(
+ model_name='machine',
+ name='id',
+ field=models.UUIDField(default=uuid.uuid7, editable=False, primary_key=True, serialize=False, unique=True),
+ ),
+ migrations.AlterField(
+ model_name='networkinterface',
+ name='id',
+ field=models.UUIDField(default=uuid.uuid7, editable=False, primary_key=True, serialize=False, unique=True),
+ ),
+ ]
diff --git a/archivebox/misc/jsonl.py b/archivebox/misc/jsonl.py
index 212ecc66..ba295cf5 100644
--- a/archivebox/misc/jsonl.py
+++ b/archivebox/misc/jsonl.py
@@ -27,10 +27,9 @@ TYPE_SNAPSHOT = 'Snapshot'
TYPE_ARCHIVERESULT = 'ArchiveResult'
TYPE_TAG = 'Tag'
TYPE_CRAWL = 'Crawl'
-TYPE_SEED = 'Seed'
TYPE_INSTALLEDBINARY = 'InstalledBinary'
-VALID_TYPES = {TYPE_SNAPSHOT, TYPE_ARCHIVERESULT, TYPE_TAG, TYPE_CRAWL, TYPE_SEED, TYPE_INSTALLEDBINARY}
+VALID_TYPES = {TYPE_SNAPSHOT, TYPE_ARCHIVERESULT, TYPE_TAG, TYPE_CRAWL, TYPE_INSTALLEDBINARY}
def parse_line(line: str) -> Optional[Dict[str, Any]]:
@@ -206,7 +205,8 @@ def crawl_to_jsonl(crawl) -> Dict[str, Any]:
return {
'type': TYPE_CRAWL,
'id': str(crawl.id),
- 'seed_id': str(crawl.seed_id),
+ 'urls': crawl.urls,
+ 'extractor': crawl.extractor,
'status': crawl.status,
'max_depth': crawl.max_depth,
'created_at': crawl.created_at.isoformat() if crawl.created_at else None,
diff --git a/archivebox/misc/logging.py b/archivebox/misc/logging.py
index 341c6c90..c571a903 100644
--- a/archivebox/misc/logging.py
+++ b/archivebox/misc/logging.py
@@ -13,9 +13,11 @@ from rich.console import Console
from rich.highlighter import Highlighter
# SETUP RICH CONSOLE / TTY detection / COLOR / PROGRESS BARS
-CONSOLE = Console()
-STDERR = Console(stderr=True)
-IS_TTY = CONSOLE.is_interactive
+# Disable wrapping - use soft_wrap=True and large width so text flows naturally
+# Colors are preserved, just no hard line breaks inserted
+CONSOLE = Console(width=32768, soft_wrap=True, force_terminal=True)
+STDERR = Console(stderr=True, width=32768, soft_wrap=True, force_terminal=True)
+IS_TTY = sys.stdout.isatty()
class RainbowHighlighter(Highlighter):
def highlight(self, text):
diff --git a/archivebox/misc/logging_util.py b/archivebox/misc/logging_util.py
index 766eed98..1016539e 100644
--- a/archivebox/misc/logging_util.py
+++ b/archivebox/misc/logging_util.py
@@ -603,21 +603,17 @@ def log_worker_event(
# Build final message
error_str = f' {type(error).__name__}: {error}' if error else ''
- # Build colored message - worker_label needs to be inside color tags
- # But first we need to format the color tags separately from the worker label
from archivebox.misc.logging import CONSOLE
from rich.text import Text
# Create a Rich Text object for proper formatting
text = Text()
- text.append(indent) # Indentation
- # Append worker label and event with color
+ text.append(indent)
text.append(f'{worker_label} {event}{error_str}', style=color)
- # Append metadata without color (add separator if metadata exists)
if metadata_str:
text.append(f' | {metadata_str}')
- CONSOLE.print(text)
+ CONSOLE.print(text, soft_wrap=True)
@enforce_types
diff --git a/archivebox/misc/monkey_patches.py b/archivebox/misc/monkey_patches.py
index 12ed05a1..2bfb7924 100644
--- a/archivebox/misc/monkey_patches.py
+++ b/archivebox/misc/monkey_patches.py
@@ -1,7 +1,5 @@
__package__ = 'archivebox'
-import sys
-import shutil
import django
import pydantic
@@ -20,14 +18,10 @@ timezone.utc = datetime.timezone.utc
# DjangoSignalWebhooksConfig.verbose_name = 'API'
-# Install rich for pretty tracebacks in console logs
-# https://rich.readthedocs.io/en/stable/traceback.html#traceback-handler
-
-from rich.traceback import install # noqa
-
-TERM_WIDTH = (shutil.get_terminal_size((200, 10)).columns - 1) if sys.stdout.isatty() else 200
-# os.environ.setdefault('COLUMNS', str(TERM_WIDTH))
-install(show_locals=True, word_wrap=False, locals_max_length=10, locals_hide_dunder=True, suppress=[django, pydantic], extra_lines=2, width=TERM_WIDTH)
+# Rich traceback handler disabled - it adds frames/boxes that wrap weirdly in log files
+# Standard Python tracebacks are used instead (full width, no frames)
+# from rich.traceback import install
+# install(show_locals=True, word_wrap=False, ...)
# Hide site-packages/sonic/client.py:115: SyntaxWarning
diff --git a/archivebox/templates/admin/progress_monitor.html b/archivebox/templates/admin/progress_monitor.html
index 1b9d9dde..3b5299af 100644
--- a/archivebox/templates/admin/progress_monitor.html
+++ b/archivebox/templates/admin/progress_monitor.html
@@ -552,21 +552,21 @@
if (crawl.status === 'queued' && !crawl.can_start) {
warningHtml = `
- β οΈ Crawl cannot start: ${crawl.seed_uri ? 'unknown error' : 'no seed URI'}
+ β οΈ Crawl cannot start: ${crawl.urls_preview ? 'unknown error' : 'no URLs'}
`;
} else if (crawl.status === 'queued' && crawl.retry_at_future) {
// Queued but retry_at is in future (was claimed by worker, will retry)
warningHtml = `
- π Retrying in ${crawl.seconds_until_retry}s...${crawl.seed_uri ? ` (${crawl.seed_uri})` : ''}
+ π Retrying in ${crawl.seconds_until_retry}s...${crawl.urls_preview ? ` (${crawl.urls_preview})` : ''}
`;
} else if (crawl.status === 'queued' && crawl.total_snapshots === 0) {
// Queued and waiting to be picked up by worker
warningHtml = `
- β³ Waiting for worker to pick up...${crawl.seed_uri ? ` (${crawl.seed_uri})` : ''}
+ β³ Waiting for worker to pick up...${crawl.urls_preview ? ` (${crawl.urls_preview})` : ''}
`;
}
@@ -577,8 +577,8 @@
metaText += ` | ${crawl.total_snapshots} snapshots`;
} else if (crawl.urls_count > 0) {
metaText += ` | ${crawl.urls_count} URLs`;
- } else if (crawl.seed_uri) {
- metaText += ` | ${crawl.seed_uri.substring(0, 40)}${crawl.seed_uri.length > 40 ? '...' : ''}`;
+ } else if (crawl.urls_preview) {
+ metaText += ` | ${crawl.urls_preview.substring(0, 40)}${crawl.urls_preview.length > 40 ? '...' : ''}`;
}
return `
diff --git a/archivebox/workers/supervisord_util.py b/archivebox/workers/supervisord_util.py
index 898f87fe..691adae4 100644
--- a/archivebox/workers/supervisord_util.py
+++ b/archivebox/workers/supervisord_util.py
@@ -26,6 +26,9 @@ CONFIG_FILE_NAME = "supervisord.conf"
PID_FILE_NAME = "supervisord.pid"
WORKERS_DIR_NAME = "workers"
+# Global reference to supervisord process for cleanup
+_supervisord_proc = None
+
ORCHESTRATOR_WORKER = {
"name": "worker_orchestrator",
"command": "archivebox manage orchestrator", # runs forever by default
@@ -78,7 +81,7 @@ def create_supervisord_config():
config_content = f"""
[supervisord]
nodaemon = true
-environment = IS_SUPERVISORD_PARENT="true"
+environment = IS_SUPERVISORD_PARENT="true",COLUMNS="200"
pidfile = {PID_FILE}
logfile = {LOG_FILE}
childlogdir = {CONSTANTS.LOGS_DIR}
@@ -143,11 +146,27 @@ def get_existing_supervisord_process():
return None
def stop_existing_supervisord_process():
+ global _supervisord_proc
SOCK_FILE = get_sock_file()
PID_FILE = SOCK_FILE.parent / PID_FILE_NAME
-
+
try:
- # if pid file exists, load PID int
+ # First try to stop via the global proc reference
+ if _supervisord_proc and _supervisord_proc.poll() is None:
+ try:
+ print(f"[π¦ΈββοΈ] Stopping supervisord process (pid={_supervisord_proc.pid})...")
+ _supervisord_proc.terminate()
+ try:
+ _supervisord_proc.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ _supervisord_proc.kill()
+ _supervisord_proc.wait(timeout=2)
+ except (BaseException, BrokenPipeError, IOError, KeyboardInterrupt):
+ pass
+ _supervisord_proc = None
+ return
+
+ # Fallback: if pid file exists, load PID int and kill that process
try:
pid = int(PID_FILE.read_text())
except (FileNotFoundError, ValueError):
@@ -156,8 +175,25 @@ def stop_existing_supervisord_process():
try:
print(f"[π¦ΈββοΈ] Stopping supervisord process (pid={pid})...")
proc = psutil.Process(pid)
+ # Kill the entire process group to ensure all children are stopped
+ children = proc.children(recursive=True)
proc.terminate()
+ # Also terminate all children
+ for child in children:
+ try:
+ child.terminate()
+ except psutil.NoSuchProcess:
+ pass
proc.wait(timeout=5)
+ # Kill any remaining children
+ for child in children:
+ try:
+ if child.is_running():
+ child.kill()
+ except psutil.NoSuchProcess:
+ pass
+ except psutil.NoSuchProcess:
+ pass
except (BaseException, BrokenPipeError, IOError, KeyboardInterrupt):
pass
finally:
@@ -174,7 +210,7 @@ def start_new_supervisord_process(daemonize=False):
LOG_FILE = CONSTANTS.LOGS_DIR / LOG_FILE_NAME
CONFIG_FILE = SOCK_FILE.parent / CONFIG_FILE_NAME
PID_FILE = SOCK_FILE.parent / PID_FILE_NAME
-
+
print(f"[π¦ΈββοΈ] Supervisord starting{' in background' if daemonize else ''}...")
pretty_log_path = pretty_path(LOG_FILE)
print(f" > Writing supervisord logs to: {pretty_log_path}")
@@ -182,50 +218,54 @@ def start_new_supervisord_process(daemonize=False):
print(f' > Using supervisord config file: {pretty_path(CONFIG_FILE)}')
print(f" > Using supervisord UNIX socket: {pretty_path(SOCK_FILE)}")
print()
-
+
# clear out existing stale state files
shutil.rmtree(WORKERS_DIR, ignore_errors=True)
PID_FILE.unlink(missing_ok=True)
get_sock_file().unlink(missing_ok=True)
CONFIG_FILE.unlink(missing_ok=True)
-
+
# create the supervisord config file
create_supervisord_config()
- # Start supervisord
- # panel = Panel(f"Starting supervisord with config: {SUPERVISORD_CONFIG_FILE}")
- # with Live(panel, refresh_per_second=1) as live:
-
- subprocess.Popen(
- f"supervisord --configuration={CONFIG_FILE}",
- stdin=None,
- shell=True,
- start_new_session=daemonize,
- )
+ # Open log file for supervisord output
+ LOG_FILE.parent.mkdir(parents=True, exist_ok=True)
+ log_handle = open(LOG_FILE, 'a')
- def exit_signal_handler(signum, frame):
- if signum == 2:
- STDERR.print("\n[π] Got Ctrl+C. Terminating child processes...")
- elif signum != 13:
- STDERR.print(f"\n[π¦ΈββοΈ] Supervisord got stop signal ({signal.strsignal(signum)}). Terminating child processes...")
- stop_existing_supervisord_process()
- raise SystemExit(0)
+ if daemonize:
+ # Start supervisord in background (daemon mode)
+ subprocess.Popen(
+ f"supervisord --configuration={CONFIG_FILE}",
+ stdin=None,
+ stdout=log_handle,
+ stderr=log_handle,
+ shell=True,
+ start_new_session=True,
+ )
+ time.sleep(2)
+ return get_existing_supervisord_process()
+ else:
+ # Start supervisord in FOREGROUND - this will block until supervisord exits
+ # supervisord with nodaemon=true will run in foreground and handle signals properly
+ # When supervisord gets SIGINT/SIGTERM, it will stop all child processes before exiting
+ proc = subprocess.Popen(
+ f"supervisord --configuration={CONFIG_FILE}",
+ stdin=None,
+ stdout=log_handle,
+ stderr=log_handle,
+ shell=True,
+ start_new_session=False, # Keep in same process group so signals propagate
+ )
- # Monitor for termination signals and cleanup child processes
- if not daemonize:
- try:
- signal.signal(signal.SIGINT, exit_signal_handler)
- signal.signal(signal.SIGHUP, exit_signal_handler)
- signal.signal(signal.SIGPIPE, exit_signal_handler)
- signal.signal(signal.SIGTERM, exit_signal_handler)
- except Exception:
- # signal handlers only work in main thread
- pass
- # otherwise supervisord will containue in background even if parent proc is ends (aka daemon mode)
+ # Store the process so we can wait on it later
+ global _supervisord_proc
+ _supervisord_proc = proc
- time.sleep(2)
+ # Wait a bit for supervisord to start up
+ time.sleep(2)
+
+ return get_existing_supervisord_process()
- return get_existing_supervisord_process()
def get_or_create_supervisord_process(daemonize=False):
SOCK_FILE = get_sock_file()
@@ -353,9 +393,15 @@ def tail_worker_logs(log_path: str):
pass
-def tail_multiple_worker_logs(log_files: list[str], follow=True):
- """Tail multiple log files simultaneously, interleaving their output."""
- import select
+def tail_multiple_worker_logs(log_files: list[str], follow=True, proc=None):
+ """Tail multiple log files simultaneously, interleaving their output.
+
+ Args:
+ log_files: List of log file paths to tail
+ follow: Whether to keep following (True) or just read existing content (False)
+ proc: Optional subprocess.Popen object - stop tailing when this process exits
+ """
+ import re
from pathlib import Path
# Convert relative paths to absolute paths
@@ -377,48 +423,53 @@ def tail_multiple_worker_logs(log_files: list[str], follow=True):
for log_path in log_paths:
try:
f = open(log_path, 'r')
- # Seek to end of file if following
- if follow:
- f.seek(0, 2) # Seek to end
- file_handles.append((log_path.name, f))
+ # Don't seek to end - show recent content so user sees something
+ # Go to end minus 4KB to show some recent logs
+ f.seek(0, 2) # Go to end first
+ file_size = f.tell()
+ if file_size > 4096:
+ f.seek(file_size - 4096)
+ f.readline() # Skip partial line
+ else:
+ f.seek(0) # Small file, read from start
+
+ file_handles.append((log_path, f))
+ print(f" [tailing {log_path.name}]")
except Exception as e:
- print(f"[yellow]Warning: Could not open {log_path}: {e}[/yellow]")
+ sys.stderr.write(f"Warning: Could not open {log_path}: {e}\n")
if not file_handles:
- print("[red]No log files could be opened[/red]")
+ sys.stderr.write("No log files could be opened\n")
return
- # Print which logs we're tailing
- log_names = [name for name, _ in file_handles]
- print(f"[dim]Tailing: {', '.join(log_names)}[/dim]")
print()
try:
while follow:
- # Read available lines from all files
- for log_name, f in file_handles:
- line = f.readline()
- if line:
- # Colorize based on log source
- if 'orchestrator' in log_name.lower():
- color = 'cyan'
- elif 'daphne' in log_name.lower():
- color = 'green'
- else:
- color = 'white'
+ # Check if the monitored process has exited
+ if proc is not None and proc.poll() is not None:
+ print(f"\n[server process exited with code {proc.returncode}]")
+ break
+ had_output = False
+ # Read ALL available lines from all files (not just one per iteration)
+ for log_path, f in file_handles:
+ while True:
+ line = f.readline()
+ if not line:
+ break # No more lines available in this file
+ had_output = True
# Strip ANSI codes if present (supervisord does this but just in case)
- import re
line_clean = re.sub(r'\x1b\[[0-9;]*m', '', line.rstrip())
-
if line_clean:
- print(f'[{color}][{log_name}][/{color}] {line_clean}')
+ print(line_clean)
- # Small sleep to avoid busy-waiting
- time.sleep(0.1)
+ # Small sleep to avoid busy-waiting (only when no output)
+ if not had_output:
+ time.sleep(0.05)
except (KeyboardInterrupt, BrokenPipeError, IOError):
- print("\n[yellow][i] Stopped tailing logs[/i][/yellow]")
+ pass # Let the caller handle the cleanup message
except SystemExit:
pass
finally:
@@ -451,6 +502,8 @@ def watch_worker(supervisor, daemon_name, interval=5):
def start_server_workers(host='0.0.0.0', port='8000', daemonize=False):
+ global _supervisord_proc
+
supervisor = get_or_create_supervisord_process(daemonize=daemonize)
bg_workers = [
@@ -466,36 +519,50 @@ def start_server_workers(host='0.0.0.0', port='8000', daemonize=False):
if not daemonize:
try:
- watch_worker(supervisor, "worker_daphne")
+ # Tail worker logs while supervisord runs
+ sys.stdout.write('Tailing worker logs (Ctrl+C to stop)...\n\n')
+ sys.stdout.flush()
+ tail_multiple_worker_logs(
+ log_files=['logs/worker_daphne.log', 'logs/worker_orchestrator.log'],
+ follow=True,
+ proc=_supervisord_proc, # Stop tailing when supervisord exits
+ )
except (KeyboardInterrupt, BrokenPipeError, IOError):
STDERR.print("\n[π] Got Ctrl+C, stopping gracefully...")
except SystemExit:
pass
except BaseException as e:
- STDERR.print(f"\n[π] Got {e.__class__.__name__} exception, stopping web server gracefully...")
- raise
+ STDERR.print(f"\n[π] Got {e.__class__.__name__} exception, stopping gracefully...")
finally:
- stop_worker(supervisor, "worker_daphne")
+ # Ensure supervisord and all children are stopped
+ stop_existing_supervisord_process()
time.sleep(0.5)
def start_cli_workers(watch=False):
+ global _supervisord_proc
+
supervisor = get_or_create_supervisord_process(daemonize=False)
start_worker(supervisor, ORCHESTRATOR_WORKER)
if watch:
try:
- watch_worker(supervisor, ORCHESTRATOR_WORKER['name'])
+ # Block on supervisord process - it will handle signals and stop children
+ if _supervisord_proc:
+ _supervisord_proc.wait()
+ else:
+ # Fallback to watching worker if no proc reference
+ watch_worker(supervisor, ORCHESTRATOR_WORKER['name'])
except (KeyboardInterrupt, BrokenPipeError, IOError):
STDERR.print("\n[π] Got Ctrl+C, stopping gracefully...")
except SystemExit:
pass
except BaseException as e:
- STDERR.print(f"\n[π] Got {e.__class__.__name__} exception, stopping orchestrator gracefully...")
- raise
+ STDERR.print(f"\n[π] Got {e.__class__.__name__} exception, stopping gracefully...")
finally:
- stop_worker(supervisor, ORCHESTRATOR_WORKER['name'])
+ # Ensure supervisord and all children are stopped
+ stop_existing_supervisord_process()
time.sleep(0.5)
return [ORCHESTRATOR_WORKER]