alex/ArchiveBox
1
0
Fork 0
mirror of https://github.com/ArchiveBox/ArchiveBox.git synced 2026-04-04 23:07:56 +10:00
Code Issues Packages Projects Releases Wiki Activity

rename OUTPUT_DIR to DATA_DIR

Browse Source
This commit is contained in:
Nick Sweeting
2024-09-30 17:44:18 -07:00
parent 363a499289
commit b913e6f426
28 changed files with 128 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
archivebox/cli/archivebox_add.py
View File

@@ -8,10 +8,11 @@ import argparse
from typing import List, Optional, IO
from ..main import add
from archivebox.misc.util import docstring
from archivebox.config import DATA_DIR, ARCHIVING_CONFIG
from ..main import add
from ..parsers import PARSERS
from ..config.legacy import OUTPUT_DIR, ONLY_NEW
from ..logging_util import SmartFormatter, accept_stdin, stderr
@@ -32,7 +33,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
parser.add_argument(
'--update', #'-u',
action='store_true',
default=not ONLY_NEW, # when ONLY_NEW=True we skip updating old links
default=not ARCHIVING_CONFIG.ONLY_NEW, # when ONLY_NEW=True we skip updating old links
help="Also retry previously skipped/failed links when adding new links",
)
parser.add_argument(
@@ -117,7 +118,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
init=command.init,
extractors=command.extract,
parser=command.parser,
out_dir=pwd or OUTPUT_DIR,
out_dir=pwd or DATA_DIR,
)

7
archivebox/cli/archivebox_config.py
View File

@@ -5,12 +5,13 @@ __command__ = 'archivebox config'
import sys
import argparse
from pathlib import Path
from typing import Optional, List, IO
from ..main import config
from archivebox.misc.util import docstring
from ..config.legacy import OUTPUT_DIR
from archivebox.config import DATA_DIR
from ..main import config
from ..logging_util import SmartFormatter, accept_stdin
@@ -56,7 +57,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
get=command.get,
set=command.set,
reset=command.reset,
out_dir=pwd or OUTPUT_DIR,
out_dir=Path(pwd) if pwd else DATA_DIR,
)

8
archivebox/cli/archivebox_help.py
View File

@@ -5,12 +5,12 @@ __command__ = 'archivebox help'
import sys
import argparse
from pathlib import Path
from typing import Optional, List, IO
from ..main import help
from archivebox.misc.util import docstring
from ..config.legacy import OUTPUT_DIR
from archivebox.config import DATA_DIR
from ..main import help
from ..logging_util import SmartFormatter, reject_stdin
@@ -25,7 +25,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
parser.parse_args(args or ())
reject_stdin(__command__, stdin)
help(out_dir=pwd or OUTPUT_DIR)
help(out_dir=Path(pwd) if pwd else DATA_DIR)
if __name__ == '__main__':

4
archivebox/cli/archivebox_init.py
View File

@@ -10,7 +10,7 @@ from typing import Optional, List, IO
from ..main import init
from archivebox.misc.util import docstring
from ..config.legacy import OUTPUT_DIR
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, reject_stdin
@@ -44,7 +44,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
force=command.force,
quick=command.quick,
setup=command.setup,
out_dir=pwd or OUTPUT_DIR,
out_dir=pwd or DATA_DIR,
)

8
archivebox/cli/archivebox_list.py
View File

@@ -5,12 +5,12 @@ __command__ = 'archivebox list'
import sys
import argparse
from pathlib import Path
from typing import Optional, List, IO
from ..main import list_all
from archivebox.misc.util import docstring
from ..config.legacy import OUTPUT_DIR
from archivebox.config import DATA_DIR
from ..main import list_all
from ..index import (
LINK_FILTERS,
get_indexed_folders,
@@ -131,7 +131,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
json=command.json,
html=command.html,
with_headers=command.with_headers,
out_dir=pwd or OUTPUT_DIR,
out_dir=Path(pwd) if pwd else DATA_DIR,
)
raise SystemExit(not matching_folders)

8
archivebox/cli/archivebox_manage.py
View File

@@ -4,19 +4,19 @@ __package__ = 'archivebox.cli'
__command__ = 'archivebox manage'
import sys
from pathlib import Path
from typing import Optional, List, IO
from ..main import manage
from archivebox.misc.util import docstring
from ..config.legacy import OUTPUT_DIR
from archivebox.config import DATA_DIR
from ..main import manage
@docstring(manage.__doc__)
def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
manage(
args=args,
out_dir=pwd or OUTPUT_DIR,
out_dir=Path(pwd) if pwd else DATA_DIR,
)

6
archivebox/cli/archivebox_oneshot.py
View File

@@ -9,10 +9,10 @@ import argparse
from pathlib import Path
from typing import List, Optional, IO
from ..main import oneshot
from archivebox.misc.util import docstring
from ..config.legacy import OUTPUT_DIR
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, accept_stdin, stderr
from ..main import oneshot
@docstring(oneshot.__doc__)
@@ -46,7 +46,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
parser.add_argument(
'--out-dir',
type=str,
default=OUTPUT_DIR,
default=DATA_DIR,
help= "Path to save the single archive folder to, e.g. ./example.com_archive"
)
command = parser.parse_args(args or ())

8
archivebox/cli/archivebox_remove.py
View File

@@ -5,13 +5,13 @@ __command__ = 'archivebox remove'
import sys
import argparse
from pathlib import Path
from typing import Optional, List, IO
from ..main import remove
from archivebox.misc.util import docstring
from ..config.legacy import OUTPUT_DIR
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, accept_stdin
from ..main import remove
@docstring(remove.__doc__)
@@ -74,7 +74,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
after=command.after,
yes=command.yes,
delete=command.delete,
out_dir=pwd or OUTPUT_DIR,
out_dir=Path(pwd) if pwd else DATA_DIR,
)

8
archivebox/cli/archivebox_schedule.py
View File

@@ -5,13 +5,13 @@ __command__ = 'archivebox schedule'
import sys
import argparse
from pathlib import Path
from typing import Optional, List, IO
from ..main import schedule
from archivebox.misc.util import docstring
from ..config.legacy import OUTPUT_DIR
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, reject_stdin
from ..main import schedule
@docstring(schedule.__doc__)
@@ -108,7 +108,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
overwrite=command.overwrite,
update=command.update,
import_path=command.import_path,
out_dir=pwd or OUTPUT_DIR,
out_dir=Path(pwd) if pwd else DATA_DIR,
)

10
archivebox/cli/archivebox_server.py
View File

@@ -5,13 +5,13 @@ __command__ = 'archivebox server'
import sys
import argparse
from pathlib import Path
from typing import Optional, List, IO
from ..main import server
from archivebox.misc.util import docstring
from ..config.legacy import OUTPUT_DIR, BIND_ADDR
from archivebox.config import DATA_DIR, SERVER_CONFIG
from ..logging_util import SmartFormatter, reject_stdin
from ..main import server
@docstring(server.__doc__)
def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
@@ -25,7 +25,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
'runserver_args',
nargs='*',
type=str,
default=[BIND_ADDR],
default=[SERVER_CONFIG.BIND_ADDR],
help='Arguments to pass to Django runserver'
)
parser.add_argument(
@@ -68,7 +68,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
init=command.init,
quick_init=command.quick_init,
createsuperuser=command.createsuperuser,
out_dir=pwd or OUTPUT_DIR,
out_dir=Path(pwd) if pwd else DATA_DIR,
)

8
archivebox/cli/archivebox_setup.py
View File

@@ -5,13 +5,13 @@ __command__ = 'archivebox setup'
import sys
import argparse
from pathlib import Path
from typing import Optional, List, IO
from ..main import setup
from archivebox.misc.util import docstring
from ..config.legacy import OUTPUT_DIR
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, reject_stdin
from ..main import setup
@docstring(setup.__doc__)
@@ -32,7 +32,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
setup(
# force=command.force,
out_dir=pwd or OUTPUT_DIR,
out_dir=Path(pwd) if pwd else DATA_DIR,
)

8
archivebox/cli/archivebox_shell.py
View File

@@ -5,13 +5,13 @@ __command__ = 'archivebox shell'
import sys
import argparse
from pathlib import Path
from typing import Optional, List, IO
from ..main import shell
from archivebox.misc.util import docstring
from ..config.legacy import OUTPUT_DIR
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, reject_stdin
from ..main import shell
@docstring(shell.__doc__)
@@ -26,7 +26,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
reject_stdin(__command__, stdin)
shell(
out_dir=pwd or OUTPUT_DIR,
out_dir=Path(pwd) if pwd else DATA_DIR,
)

8
archivebox/cli/archivebox_status.py
View File

@@ -5,13 +5,13 @@ __command__ = 'archivebox status'
import sys
import argparse
from pathlib import Path
from typing import Optional, List, IO
from ..main import status
from archivebox.misc.util import docstring
from ..config.legacy import OUTPUT_DIR
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, reject_stdin
from ..main import status
@docstring(status.__doc__)
@@ -25,7 +25,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
parser.parse_args(args or ())
reject_stdin(__command__, stdin)
status(out_dir=pwd or OUTPUT_DIR)
status(out_dir=Path(pwd) if pwd else DATA_DIR)
if __name__ == '__main__':

8
archivebox/cli/archivebox_update.py
View File

@@ -5,12 +5,11 @@ __command__ = 'archivebox update'
import sys
import argparse
from pathlib import Path
from typing import List, Optional, IO
from ..main import update
from archivebox.misc.util import docstring
from ..config.legacy import OUTPUT_DIR
from archivebox.config import DATA_DIR
from ..index import (
LINK_FILTERS,
get_indexed_folders,
@@ -25,6 +24,7 @@ from ..index import (
get_unrecognized_folders,
)
from ..logging_util import SmartFormatter, accept_stdin
from ..main import update
@docstring(update.__doc__)
@@ -127,7 +127,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
status=command.status,
after=command.after,
before=command.before,
out_dir=pwd or OUTPUT_DIR,
out_dir=Path(pwd) if pwd else DATA_DIR,
extractors=command.extract,
)

8
archivebox/cli/archivebox_version.py
View File

@@ -5,13 +5,13 @@ __command__ = 'archivebox version'
import sys
import argparse
from pathlib import Path
from typing import Optional, List, IO
from ..main import version
from archivebox.misc.util import docstring
from ..config.legacy import OUTPUT_DIR
from archivebox.config import DATA_DIR
from ..logging_util import SmartFormatter, reject_stdin
from ..main import version
@docstring(version.__doc__)
@@ -32,7 +32,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
version(
quiet=command.quiet,
out_dir=pwd or OUTPUT_DIR,
out_dir=Path(pwd) if pwd else DATA_DIR,
)

52
archivebox/cli/tests.py
View File

@@ -15,7 +15,7 @@ TEST_CONFIG = {
'USE_COLOR': 'False',
'SHOW_PROGRESS': 'False',
'OUTPUT_DIR': 'data.tests',
'DATA_DIR': 'data.tests',
'SAVE_ARCHIVE_DOT_ORG': 'False',
'SAVE_TITLE': 'False',
@@ -27,12 +27,12 @@ TEST_CONFIG = {
'USE_YOUTUBEDL': 'False',
}
OUTPUT_DIR = 'data.tests'
DATA_DIR = 'data.tests'
os.environ.update(TEST_CONFIG)
from ..main import init
from ..index import load_main_index
from ..config.legacy import (
from archivebox.config.constants import (
SQL_INDEX_FILENAME,
JSON_INDEX_FILENAME,
HTML_INDEX_FILENAME,
@@ -101,22 +101,22 @@ def output_hidden(show_failing=True):
class TestInit(unittest.TestCase):
def setUp(self):
os.makedirs(OUTPUT_DIR, exist_ok=True)
os.makedirs(DATA_DIR, exist_ok=True)
def tearDown(self):
shutil.rmtree(OUTPUT_DIR, ignore_errors=True)
shutil.rmtree(DATA_DIR, ignore_errors=True)
def test_basic_init(self):
with output_hidden():
archivebox_init.main([])
assert (Path(OUTPUT_DIR) / SQL_INDEX_FILENAME).exists()
assert (Path(OUTPUT_DIR) / JSON_INDEX_FILENAME).exists()
assert (Path(OUTPUT_DIR) / HTML_INDEX_FILENAME).exists()
assert len(load_main_index(out_dir=OUTPUT_DIR)) == 0
assert (Path(DATA_DIR) / SQL_INDEX_FILENAME).exists()
assert (Path(DATA_DIR) / JSON_INDEX_FILENAME).exists()
assert (Path(DATA_DIR) / HTML_INDEX_FILENAME).exists()
assert len(load_main_index(out_dir=DATA_DIR)) == 0
def test_conflicting_init(self):
with open(Path(OUTPUT_DIR) / 'test_conflict.txt', 'w+', encoding='utf-8') as f:
with open(Path(DATA_DIR) / 'test_conflict.txt', 'w+', encoding='utf-8') as f:
f.write('test')
try:
@@ -126,11 +126,11 @@ class TestInit(unittest.TestCase):
except SystemExit:
pass
assert not (Path(OUTPUT_DIR) / SQL_INDEX_FILENAME).exists()
assert not (Path(OUTPUT_DIR) / JSON_INDEX_FILENAME).exists()
assert not (Path(OUTPUT_DIR) / HTML_INDEX_FILENAME).exists()
assert not (Path(DATA_DIR) / SQL_INDEX_FILENAME).exists()
assert not (Path(DATA_DIR) / JSON_INDEX_FILENAME).exists()
assert not (Path(DATA_DIR) / HTML_INDEX_FILENAME).exists()
try:
load_main_index(out_dir=OUTPUT_DIR)
load_main_index(out_dir=DATA_DIR)
assert False, 'load_main_index should raise an exception when no index is present'
except Exception:
pass
@@ -138,36 +138,36 @@ class TestInit(unittest.TestCase):
def test_no_dirty_state(self):
with output_hidden():
init()
shutil.rmtree(OUTPUT_DIR, ignore_errors=True)
shutil.rmtree(DATA_DIR, ignore_errors=True)
with output_hidden():
init()
class TestAdd(unittest.TestCase):
def setUp(self):
os.makedirs(OUTPUT_DIR, exist_ok=True)
os.makedirs(DATA_DIR, exist_ok=True)
with output_hidden():
init()
def tearDown(self):
shutil.rmtree(OUTPUT_DIR, ignore_errors=True)
shutil.rmtree(DATA_DIR, ignore_errors=True)
def test_add_arg_url(self):
with output_hidden():
archivebox_add.main(['https://getpocket.com/users/nikisweeting/feed/all'])
all_links = load_main_index(out_dir=OUTPUT_DIR)
all_links = load_main_index(out_dir=DATA_DIR)
assert len(all_links) == 30
def test_add_arg_file(self):
test_file = Path(OUTPUT_DIR) / 'test.txt'
test_file = Path(DATA_DIR) / 'test.txt'
with open(test_file, 'w+', encoding='utf') as f:
f.write(test_urls)
with output_hidden():
archivebox_add.main([test_file])
all_links = load_main_index(out_dir=OUTPUT_DIR)
all_links = load_main_index(out_dir=DATA_DIR)
assert len(all_links) == 12
os.remove(test_file)
@@ -175,40 +175,40 @@ class TestAdd(unittest.TestCase):
with output_hidden():
archivebox_add.main([], stdin=test_urls)
all_links = load_main_index(out_dir=OUTPUT_DIR)
all_links = load_main_index(out_dir=DATA_DIR)
assert len(all_links) == 12
class TestRemove(unittest.TestCase):
def setUp(self):
os.makedirs(OUTPUT_DIR, exist_ok=True)
os.makedirs(DATA_DIR, exist_ok=True)
with output_hidden():
init()
archivebox_add.main([], stdin=test_urls)
# def tearDown(self):
# shutil.rmtree(OUTPUT_DIR, ignore_errors=True)
# shutil.rmtree(DATA_DIR, ignore_errors=True)
def test_remove_exact(self):
with output_hidden():
archivebox_remove.main(['--yes', '--delete', 'https://example5.com/'])
all_links = load_main_index(out_dir=OUTPUT_DIR)
all_links = load_main_index(out_dir=DATA_DIR)
assert len(all_links) == 11
def test_remove_regex(self):
with output_hidden():
archivebox_remove.main(['--yes', '--delete', '--filter-type=regex', r'http(s)?:\/\/(.+\.)?(example\d\.com)'])
all_links = load_main_index(out_dir=OUTPUT_DIR)
all_links = load_main_index(out_dir=DATA_DIR)
assert len(all_links) == 4
def test_remove_domain(self):
with output_hidden():
archivebox_remove.main(['--yes', '--delete', '--filter-type=domain', 'example5.com', 'example6.com'])
all_links = load_main_index(out_dir=OUTPUT_DIR)
all_links = load_main_index(out_dir=DATA_DIR)
assert len(all_links) == 10
def test_remove_none(self):