move docstrings to main.py out of cli files

archivebox/cli/__init__.py

@@ -33,9 +33,8 @@ def list_subcommands() -> Dict[str, str]:
             subcommand = filename.replace('archivebox_', '').replace('.py', '')
             module = import_module('.archivebox_{}'.format(subcommand), __package__)
             assert is_valid_cli_module(module, subcommand)
-            COMMANDS.append((subcommand, module.__description__))  # type: ignore
+            COMMANDS.append((subcommand, module.main.__doc__))
             globals()[subcommand] = module.main
-            module.main.__doc__ = module.__description__
 
     display_order = lambda cmd: (
         display_first.index(cmd[0])
@@ -50,7 +49,7 @@ def run_subcommand(subcommand: str,
                    subcommand_args: List[str]=None,
                    stdin: Optional[IO]=None,
                    pwd: Optional[str]=None) -> None:
-    """run a given ArchiveBox subcommand with the given list of args"""
+    """Run a given ArchiveBox subcommand with the given list of args"""
 
     module = import_module('.archivebox_{}'.format(subcommand), __package__)
     module.main(args=subcommand_args, stdin=stdin, pwd=pwd)    # type: ignore

archivebox/cli/archivebox.py

@@ -3,7 +3,6 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox'
-__description__ = 'ArchiveBox: The self-hosted internet archive.'
 
 import sys
 import argparse
@@ -18,7 +17,7 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
     subcommands = list_subcommands()
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description='ArchiveBox: The self-hosted internet archive',
         add_help=False,
     )
     group = parser.add_mutually_exclusive_group()

archivebox/cli/archivebox_add.py

@@ -2,29 +2,29 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox add'
-__description__ = 'Add a new URL or list of URLs to your archive'
 
 import sys
 import argparse
 
 from typing import List, Optional, IO
 
-from ..main import add
-from ..util import SmartFormatter, accept_stdin
+from ..main import add, docstring
 from ..config import OUTPUT_DIR, ONLY_NEW
+from .logging import SmartFormatter, accept_stdin
 
 
+@docstring(add.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=add.__doc__,
         add_help=True,
         formatter_class=SmartFormatter,
     )
     parser.add_argument(
         '--update-all', #'-n',
         action='store_true',
-        default=not ONLY_NEW,
+        default=not ONLY_NEW,  # when ONLY_NEW=True we skip updating old links
         help="Also retry previously skipped/failed links when adding new links",
     )
     parser.add_argument(

archivebox/cli/archivebox_config.py

@@ -2,22 +2,22 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox config'
-__description__ = 'Get and set your ArchiveBox project configuration values'
 
 import sys
 import argparse
 
 from typing import Optional, List, IO
 
-from ..main import config
-from ..util import SmartFormatter, accept_stdin
+from ..main import config, docstring
 from ..config import OUTPUT_DIR
+from .logging import SmartFormatter, accept_stdin
 
 
+@docstring(config.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=config.__doc__,
         add_help=True,
         formatter_class=SmartFormatter,
     )

archivebox/cli/archivebox_help.py

@@ -2,23 +2,24 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox help'
-__description__ = 'Print the ArchiveBox help message and usage'
 
 import sys
 import argparse
 
 from typing import Optional, List, IO
 
-from ..main import help
-from ..util import reject_stdin
+from ..main import help, docstring
 from ..config import OUTPUT_DIR
+from .logging import SmartFormatter, reject_stdin
 
 
+@docstring(help.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=help.__doc__,
         add_help=True,
+        formatter_class=SmartFormatter,
     )
     parser.parse_args(args or ())
     reject_stdin(__command__, stdin)

archivebox/cli/archivebox_info.py

@@ -2,23 +2,24 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox info'
-__description__ = 'Print out some info and statistics about the archive collection'
 
 import sys
 import argparse
 
 from typing import Optional, List, IO
 
-from ..main import info
+from ..main import info, docstring
 from ..config import OUTPUT_DIR
-from ..util import reject_stdin
+from .logging import SmartFormatter, reject_stdin
 
 
+@docstring(info.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=info.__doc__,
         add_help=True,
+        formatter_class=SmartFormatter,
     )
     parser.parse_args(args or ())
     reject_stdin(__command__, stdin)

archivebox/cli/archivebox_init.py

@@ -2,23 +2,24 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox init'
-__description__ = 'Initialize a new ArchiveBox collection in the current directory'
 
 import sys
 import argparse
 
 from typing import Optional, List, IO
 
-from ..main import init
-from ..util import reject_stdin
+from ..main import init, docstring
 from ..config import OUTPUT_DIR
+from .logging import SmartFormatter, reject_stdin
 
 
+@docstring(init.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=init.__doc__,
         add_help=True,
+        formatter_class=SmartFormatter,
     )
     parser.parse_args(args or ())
     reject_stdin(__command__, stdin)

archivebox/cli/archivebox_list.py

@@ -2,15 +2,13 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox list'
-__description__ = 'List, filter, and export information about archive entries'
 
 import sys
 import argparse
 
 from typing import Optional, List, IO
 
-from ..main import list_all
-from ..util import SmartFormatter, accept_stdin
+from ..main import list_all, docstring
 from ..config import OUTPUT_DIR
 from ..index import (
     get_indexed_folders,
@@ -24,11 +22,14 @@ from ..index import (
     get_corrupted_folders,
     get_unrecognized_folders,
 )
+from .logging import SmartFormatter, accept_stdin
 
+
+@docstring(list_all.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=list_all.__doc__,
         add_help=True,
         formatter_class=SmartFormatter,
     )

archivebox/cli/archivebox_manage.py

@@ -2,16 +2,16 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox manage'
-__description__ = 'Run an ArchiveBox Django management command'
 
 import sys
 
 from typing import Optional, List, IO
 
-from ..main import manage
+from ..main import manage, docstring
 from ..config import OUTPUT_DIR
 
 
+@docstring(manage.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     manage(
         args=args,

archivebox/cli/archivebox_remove.py

@@ -2,23 +2,24 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox remove'
-__description__ = 'Remove the specified URLs from the archive.'
 
 import sys
 import argparse
 
 from typing import Optional, List, IO
 
-from ..main import remove
-from ..util import accept_stdin
+from ..main import remove, docstring
 from ..config import OUTPUT_DIR
+from .logging import SmartFormatter, accept_stdin
 
 
+@docstring(remove.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=remove.__doc__,
         add_help=True,
+        formatter_class=SmartFormatter,
     )
     parser.add_argument(
         '--yes', # '-y',

archivebox/cli/archivebox_schedule.py

@@ -2,23 +2,24 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox schedule'
-__description__ = 'Set ArchiveBox to regularly import URLs at specific times using cron'
 
 import sys
 import argparse
 
 from typing import Optional, List, IO
 
-from ..main import schedule
-from ..util import reject_stdin
+from ..main import schedule, docstring
 from ..config import OUTPUT_DIR
+from .logging import SmartFormatter, reject_stdin
 
 
+@docstring(schedule.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=schedule.__doc__,
         add_help=True,
+        formatter_class=SmartFormatter,
     )
     parser.add_argument(
         '--quiet', '-q',

archivebox/cli/archivebox_server.py

@@ -2,23 +2,24 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox server'
-__description__ = 'Run the ArchiveBox HTTP server'
 
 import sys
 import argparse
 
 from typing import Optional, List, IO
 
-from ..main import server
-from ..util import reject_stdin
+from ..main import server, docstring
 from ..config import OUTPUT_DIR
+from .logging import SmartFormatter, reject_stdin
 
 
+@docstring(server.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=server.__doc__,
         add_help=True,
+        formatter_class=SmartFormatter,
     )
     parser.add_argument(
         'runserver_args',

archivebox/cli/archivebox_shell.py

@@ -2,23 +2,24 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox shell'
-__description__ = 'Enter an interactive ArchiveBox Django shell'
 
 import sys
 import argparse
 
 from typing import Optional, List, IO
 
-from ..main import shell
+from ..main import shell, docstring
 from ..config import OUTPUT_DIR
-from ..util import reject_stdin
+from .logging import SmartFormatter, reject_stdin
 
 
+@docstring(shell.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=shell.__doc__,
         add_help=True,
+        formatter_class=SmartFormatter,
     )
     parser.parse_args(args or ())
     reject_stdin(__command__, stdin)

archivebox/cli/archivebox_update.py

@@ -2,15 +2,13 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox update'
-__description__ = 'Import any new links from subscriptions and retry any previously failed/skipped links'
 
 import sys
 import argparse
 
 from typing import List, Optional, IO
 
-from ..main import update
-from ..util import SmartFormatter, accept_stdin
+from ..main import update, docstring
 from ..config import OUTPUT_DIR
 from ..index import (
     get_indexed_folders,
@@ -24,12 +22,14 @@ from ..index import (
     get_corrupted_folders,
     get_unrecognized_folders,
 )
+from .logging import SmartFormatter, accept_stdin
 
 
+@docstring(update.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=update.__doc__,
         add_help=True,
         formatter_class=SmartFormatter,
     )
@@ -99,9 +99,9 @@ def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional
         nargs='*',
         type=str,
         default=None,
-        help='List only URLs matching these filter patterns.'
+        help='Update only URLs matching these filter patterns.'
     )
-    command = parser.parse_args(args)
+    command = parser.parse_args(args or ())
     filter_patterns_str = accept_stdin(stdin)
 
     update(

archivebox/cli/archivebox_version.py

@@ -2,23 +2,24 @@
 
 __package__ = 'archivebox.cli'
 __command__ = 'archivebox version'
-__description__ = 'Print the ArchiveBox version and dependency information'
 
 import sys
 import argparse
 
 from typing import Optional, List, IO
 
-from ..main import version
-from ..util import reject_stdin
+from ..main import version, docstring
 from ..config import OUTPUT_DIR
+from .logging import SmartFormatter, reject_stdin
 
 
+@docstring(version.__doc__)
 def main(args: Optional[List[str]]=None, stdin: Optional[IO]=None, pwd: Optional[str]=None) -> None:
     parser = argparse.ArgumentParser(
         prog=__command__,
-        description=__description__,
+        description=version.__doc__,
         add_help=True,
+        formatter_class=SmartFormatter,
     )
     parser.add_argument(
         '--quiet', '-q',