mirror of
https://github.com/ArchiveBox/ArchiveBox.git
synced 2026-03-18 14:03:54 +10:00
add custom WGET_USER_AGENT override option
This commit is contained in:
2
fetch.py
2
fetch.py
@@ -17,6 +17,7 @@ from config import (
|
||||
FETCH_AUDIO,
|
||||
FETCH_VIDEO,
|
||||
FETCH_FAVICON,
|
||||
WGET_USER_AGENT,
|
||||
TIMEOUT,
|
||||
ANSI,
|
||||
progress,
|
||||
@@ -32,6 +33,7 @@ def fetch_wget(out_dir, link, overwrite=False, requisites=True, timeout=TIMEOUT)
|
||||
CMD = [
|
||||
*'wget --timestamping --adjust-extension --no-parent'.split(' '), # Docs: https://www.gnu.org/software/wget/manual/wget.html
|
||||
*(('--page-requisites', '--convert-links') if requisites else ()),
|
||||
*(('--user-agent="{}"'.format(WGET_USER_AGENT)) if WGET_USER_AGENT else ()),
|
||||
link['url'],
|
||||
]
|
||||
end = progress(timeout, prefix=' ')
|
||||
|
||||
Reference in New Issue
Block a user