improve config loading of TMP_DIR, LIB_DIR, move to separate files

archivebox/misc/checks.py

@@ -1,37 +1,44 @@
 __package__ = 'archivebox.misc'
 
-from archivebox.config import DATA_DIR, ARCHIVE_DIR, CONSTANTS, SHELL_CONFIG
+import sys
+from rich import print
 
-from .logging import stderr
+# DO NOT ADD ANY TOP-LEVEL IMPORTS HERE
+# this file is imported by archivebox/__init__.py
+# and any imports here will be imported by EVERYTHING else
+# so this file should only be used for pure python checks
+# that don't need to import other parts of ArchiveBox
 
 
 def check_data_folder() -> None:
-
+    from archivebox import DATA_DIR, ARCHIVE_DIR
+    
     archive_dir_exists = ARCHIVE_DIR.exists()
     if not archive_dir_exists:
-        stderr('[X] No archivebox index found in the current directory.', color='red')
-        stderr(f'    {DATA_DIR}', color='lightyellow')
-        stderr()
-        stderr('    {lightred}Hint{reset}: Are you running archivebox in the right folder?'.format(**SHELL_CONFIG.ANSI))
-        stderr('        cd path/to/your/archive/folder')
-        stderr('        archivebox [command]')
-        stderr()
-        stderr('    {lightred}Hint{reset}: To create a new archive collection or import existing data in this folder, run:'.format(**SHELL_CONFIG.ANSI))
-        stderr('        archivebox init')
+        print('[red][X] No archivebox index found in the current directory.[/red]', file=sys.stderr)
+        print(f'    {DATA_DIR}', file=sys.stderr)
+        print(file=sys.stderr)
+        print('    [violet]Hint[/violet]: Are you running archivebox in the right folder?', file=sys.stderr)
+        print('        cd path/to/your/archive/folder', file=sys.stderr)
+        print('        archivebox [command]', file=sys.stderr)
+        print(file=sys.stderr)
+        print('    [violet]Hint[/violet]: To create a new archive collection or import existing data in this folder, run:', file=sys.stderr)
+        print('        archivebox init', file=sys.stderr)
         raise SystemExit(2)
-
-
+    
+    
 def check_migrations():
+    from archivebox import DATA_DIR, CONSTANTS
     from ..index.sql import list_migrations
 
     pending_migrations = [name for status, name in list_migrations() if not status]
 
     if pending_migrations:
-        stderr('[X] This collection was created with an older version of ArchiveBox and must be upgraded first.', color='lightyellow')
-        stderr(f'    {DATA_DIR}')
-        stderr()
-        stderr(f'    To upgrade it to the latest version and apply the {len(pending_migrations)} pending migrations, run:')
-        stderr('        archivebox init')
+        print('[red][X] This collection was created with an older version of ArchiveBox and must be upgraded first.[/red]')
+        print(f'    {DATA_DIR}', file=sys.stderr)
+        print(file=sys.stderr)
+        print(f'    [violet]Hint:[/violet] To upgrade it to the latest version and apply the {len(pending_migrations)} pending migrations, run:', file=sys.stderr)
+        print('        archivebox init', file=sys.stderr)
         raise SystemExit(3)
 
     CONSTANTS.SOURCES_DIR.mkdir(exist_ok=True)
@@ -39,3 +46,39 @@ def check_migrations():
     # CONSTANTS.CACHE_DIR.mkdir(exist_ok=True)
     (CONSTANTS.LIB_DIR / 'bin').mkdir(exist_ok=True, parents=True)
     (CONSTANTS.PERSONAS_DIR / 'Default').mkdir(exist_ok=True, parents=True)
+
+
+def check_io_encoding():
+    PYTHON_ENCODING = (sys.__stdout__ or sys.stdout or sys.__stderr__ or sys.stderr).encoding.upper().replace('UTF8', 'UTF-8')
+            
+    if PYTHON_ENCODING != 'UTF-8':
+        print(f'[red][X] Your system is running python3 scripts with a bad locale setting: {PYTHON_ENCODING} (it should be UTF-8).[/red]', file=sys.stderr)
+        print('    To fix it, add the line "export PYTHONIOENCODING=UTF-8" to your ~/.bashrc file (without quotes)', file=sys.stderr)
+        print('    Or if you\'re using ubuntu/debian, run "dpkg-reconfigure locales"', file=sys.stderr)
+        print('')
+        print('    Confirm that it\'s fixed by opening a new shell and running:', file=sys.stderr)
+        print('        python3 -c "import sys; print(sys.stdout.encoding)"   # should output UTF-8', file=sys.stderr)
+        raise SystemExit(2)
+
+
+def check_not_root():
+    from archivebox.config.permissions import IS_ROOT, IN_DOCKER
+    
+    attempted_command = ' '.join(sys.argv[1:]) if len(sys.argv) > 1 else ''
+    is_getting_help = '-h' in sys.argv or '--help' in sys.argv or 'help' in sys.argv[:2]
+    is_getting_version = '--version' in sys.argv or 'version' in sys.argv[:2]
+    is_installing = 'setup' in sys.argv[:2] or 'install' in sys.argv[:2]
+    
+    if IS_ROOT and not (is_getting_help or is_getting_version or is_installing):
+        print('[red][!] ArchiveBox should never be run as root![/red]', file=sys.stderr)
+        print('    For more information, see the security overview documentation:', file=sys.stderr)
+        print('        https://github.com/ArchiveBox/ArchiveBox/wiki/Security-Overview#do-not-run-as-root', file=sys.stderr)
+        
+        if IN_DOCKER:
+            print('[red][!] When using Docker, you must run commands with [green]docker run[/green] instead of [yellow3]docker exec[/yellow3], e.g.:', file=sys.stderr)
+            print('        docker compose run archivebox {attempted_command}', file=sys.stderr)
+            print(f'        docker run -it -v $PWD/data:/data archivebox/archivebox {attempted_command}', file=sys.stderr)
+            print('        or:', file=sys.stderr)
+            print(f'        docker compose exec --user=archivebox archivebox /bin/bash -c "archivebox {attempted_command}"', file=sys.stderr)
+            print(f'        docker exec -it --user=archivebox <container id> /bin/bash -c "archivebox {attempted_command}"', file=sys.stderr)
+        raise SystemExit(2)

archivebox/misc/logging.py

@@ -13,6 +13,7 @@ from rich.highlighter import Highlighter
 
 # SETUP RICH CONSOLE / TTY detection / COLOR / PROGRESS BARS
 CONSOLE = Console()
+STDERR = Console(stderr=True)
 IS_TTY = CONSOLE.is_interactive
 
 
@@ -51,7 +52,7 @@ COLOR_DICT = defaultdict(lambda: [(0, 0, 0), (0, 0, 0)], {
     '37': [(255, 255, 255), (255, 255, 255)],
 })
 
-# Logging Helpers
+# Logging Helpers (DEPRECATED, use rich.print instead going forward)
 def stdout(*args, color: Optional[str]=None, prefix: str='', config: Optional[benedict]=None) -> None:
     ansi = DEFAULT_CLI_COLORS if (config or {}).get('USE_COLOR') else ANSI
 

archivebox/misc/system.py

@@ -4,7 +4,6 @@ __package__ = 'archivebox.misc'
 import os
 import signal
 import shutil
-import getpass
 
 from json import dump
 from pathlib import Path
@@ -14,7 +13,7 @@ from subprocess import _mswindows, PIPE, Popen, CalledProcessError, CompletedPro
 from crontab import CronTab
 from atomicwrites import atomic_write as lib_atomic_write
 
-from archivebox.config import STORAGE_CONFIG
+from archivebox.config.common import STORAGE_CONFIG
 from archivebox.misc.util import enforce_types, ExtendedEncoder
 
 

archivebox/misc/util.py

@@ -1,4 +1,4 @@
-__package__ = 'archivebox'
+__package__ = 'archivebox.misc'
 
 import re
 import requests
@@ -25,10 +25,10 @@ except ImportError:
     detect_encoding = lambda rawdata: "utf-8"
 
 
-from archivebox.config.constants import STATICFILE_EXTENSIONS
-from archivebox.config import ARCHIVING_CONFIG
+from archivebox.config import CONSTANTS
+from archivebox.config.common import ARCHIVING_CONFIG
 
-from .misc.logging import COLOR_DICT
+from .logging import COLOR_DICT
 
 
 ### Parsing Helpers
@@ -120,7 +120,7 @@ def find_all_urls(urls_str: str):
 
 def is_static_file(url: str):
     # TODO: the proper way is with MIME type detection + ext, not only extension
-    return extension(url).lower() in STATICFILE_EXTENSIONS
+    return extension(url).lower() in CONSTANTS.STATICFILE_EXTENSIONS
 
 
 def enforce_types(func):