alex/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-04-06 07:47:53 +10:00
Code Issues Packages Projects Releases Wiki Activity

finish migrating almost all config to new system

Browse Source
This commit is contained in:
Nick Sweeting
2024-09-30 23:21:34 -07:00
parent 4b6a2a3e50
commit d21bc86075
25 changed files with 246 additions and 349 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
archivebox/extractors/headers.py
View File

@@ -4,23 +4,14 @@ from pathlib import Path
from typing import Optional
from ..index.schema import Link, ArchiveResult, ArchiveOutput
from archivebox.misc.system import atomic_write
from archivebox.misc.util import (
enforce_types,
get_headers,
dedupe,
)
from ..config.legacy import (
TIMEOUT,
CURL_BINARY,
CURL_ARGS,
CURL_EXTRA_ARGS,
CURL_USER_AGENT,
CURL_VERSION,
CHECK_SSL_VALIDITY,
SAVE_HEADERS
)
from archivebox.plugins_extractor.curl.apps import CURL_CONFIG, CURL_BINARY
from ..index.schema import Link, ArchiveResult, ArchiveOutput
from ..logging_util import TimedProgress
def get_output_path():
@@ -29,34 +20,38 @@ def get_output_path():
@enforce_types
def should_save_headers(link: Link, out_dir: Optional[str]=None, overwrite: Optional[bool]=False) -> bool:
out_dir = out_dir or Path(link.link_dir)
if not overwrite and (out_dir / get_output_path()).exists():
out_dir_path = Path(out_dir or link.link_dir)
assert out_dir_path
if not overwrite and (out_dir_path / get_output_path()).exists():
return False
return SAVE_HEADERS
return CURL_CONFIG.SAVE_HEADERS
@enforce_types
def save_headers(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT) -> ArchiveResult:
def save_headers(link: Link, out_dir: Optional[str]=None, timeout: int=CURL_CONFIG.CURL_TIMEOUT) -> ArchiveResult:
"""Download site headers"""
out_dir = Path(out_dir or link.link_dir)
output_folder = out_dir.absolute()
curl_binary = CURL_BINARY.load()
assert curl_binary.abspath and curl_binary.version
out_dir_path = Path(out_dir or link.link_dir)
output_folder = out_dir_path.absolute()
output: ArchiveOutput = get_output_path()
status = 'succeeded'
timer = TimedProgress(timeout, prefix=' ')
timer = TimedProgress(timeout + 1, prefix=' ')
# later options take precedence
options = [
*CURL_ARGS,
*CURL_EXTRA_ARGS,
*CURL_CONFIG.CURL_ARGS,
*CURL_CONFIG.CURL_EXTRA_ARGS,
'--head',
'--max-time', str(timeout),
*(['--user-agent', '{}'.format(CURL_USER_AGENT)] if CURL_USER_AGENT else []),
*([] if CHECK_SSL_VALIDITY else ['--insecure']),
*(['--user-agent', '{}'.format(CURL_CONFIG.CURL_USER_AGENT)] if CURL_CONFIG.CURL_USER_AGENT else []),
*([] if CURL_CONFIG.CURL_CHECK_SSL_VALIDITY else ['--insecure']),
]
cmd = [
CURL_BINARY,
str(curl_binary.abspath),
*dedupe(options),
link.url,
]
@@ -72,8 +67,8 @@ def save_headers(link: Link, out_dir: Optional[str]=None, timeout: int=TIMEOUT)
return ArchiveResult(
cmd=cmd,
pwd=str(out_dir),
cmd_version=CURL_VERSION,
pwd=str(out_dir_path),
cmd_version=str(curl_binary.version),
output=output,
status=status,
**timer.stats,