move to new vendoring fallback logic

archivebox/config.py

@@ -52,6 +52,11 @@ from .config_stubs import (
     ConfigDefaultDict,
 )
 
+# load fallback libraries from vendor dir
+from .vendor import load_vendored_libs
+load_vendored_libs()
+
+
 
 ############################### Config Schema ##################################
 

archivebox/core/forms.py

@@ -4,7 +4,7 @@ from django import forms
 
 from ..util import URL_REGEX
 from ..parsers import PARSERS
-from ..vendor.taggit_utils import edit_string_for_tags, parse_tags
+from taggit.utils import edit_string_for_tags, parse_tags
 
 PARSER_CHOICES = [
     (parser_key, parser[0])

archivebox/parsers/pocket_api.py

@@ -7,7 +7,7 @@ from typing import IO, Iterable, Optional
 from configparser import ConfigParser
 
 from pathlib import Path
-from ..vendor.pocket import Pocket
+from pocket import Pocket
 
 from ..index.schema import Link
 from ..util import enforce_types

archivebox/system.py

@@ -11,13 +11,12 @@ from typing import Optional, Union, Set, Tuple
 from subprocess import _mswindows, PIPE, Popen, CalledProcessError, CompletedProcess, TimeoutExpired
 
 from crontab import CronTab
-from .vendor.atomicwrites import atomic_write as lib_atomic_write
+from atomicwrites import atomic_write as lib_atomic_write
 
 from .util import enforce_types, ExtendedEncoder
 from .config import PYTHON_BINARY, OUTPUT_PERMISSIONS, DIR_OUTPUT_PERMISSIONS, ENFORCE_ATOMIC_WRITES
 
 
-
 def run(cmd, *args, input=None, capture_output=True, timeout=None, check=False, text=False, start_new_session=True, **kwargs):
     """Patched of subprocess.run to kill forked child subprocesses and fix blocking io making timeout=innefective
         Mostly copied from https://github.com/python/cpython/blob/master/Lib/subprocess.py

archivebox/util.py

@@ -16,7 +16,7 @@ from datetime import datetime, timezone
 from dateparser import parse as dateparser
 from requests.exceptions import RequestException, ReadTimeout
 
-from .vendor.base32_crockford import encode as base32_encode                            # type: ignore
+from base32_crockford import encode as base32_encode                            # type: ignore
 from w3lib.encoding import html_body_declared_encoding, http_content_type_encoding
 from os.path import lexists
 from os import remove as remove_file

archivebox/vendor/__init__.py

@@ -0,0 +1,34 @@
+import sys
+import inspect
+import importlib
+from pathlib import Path
+
+VENDOR_DIR = Path(__file__).parent
+
+VENDORED_LIBS = {
+    # sys.path dir:         library name
+    'python-atomicwrites':  'atomicwrites',
+    'django-taggit':        'taggit',
+    'pydantic-pkgr':        'pydantic_pkgr',
+    'pocket':               'pocket',
+    'base32-crockford':     'base32_crockford',
+}
+
+def load_vendored_libs():
+    for lib_subdir, lib_name in VENDORED_LIBS.items():
+        lib_dir = VENDOR_DIR / lib_subdir
+        assert lib_dir.is_dir(), 'Expected vendor libary {lib_name} could not be found in {lib_dir}'
+
+        try:
+            lib = importlib.import_module(lib_name)
+            # print(f"Successfully imported lib from environment {lib_name}: {inspect.getfile(lib)}")
+        except ImportError:
+            sys.path.append(str(lib_dir))
+            try:
+                lib = importlib.import_module(lib_name)
+                # print(f"Successfully imported lib from vendored fallback {lib_name}: {inspect.getfile(lib)}")
+            except ImportError as e:
+                print(f"Failed to import lib from environment or vendored fallback {lib_name}: {e}", file=sys.stderr)
+                sys.exit(1)
+        
+

archivebox/vendor/atomicwrites.py

@@ -1 +0,0 @@
-python-atomicwrites/atomicwrites/__init__.py

archivebox/vendor/base32_crockford.py

@@ -1 +0,0 @@
-base32-crockford/base32_crockford.py

archivebox/vendor/package-lock.json

@@ -1 +0,0 @@
-../../package-lock.json

archivebox/vendor/package.json

@@ -1 +0,0 @@
-../../package.json

archivebox/vendor/pocket.py

@@ -1 +0,0 @@
-pocket/pocket.py

archivebox/vendor/taggit_utils.py

@@ -1 +0,0 @@
-django-taggit/taggit/utils.py